From 40c9103f95cd870e83760383f5aa17f811ab9a90 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 14 Mar 2025 21:22:12 +0100 Subject: [PATCH 01/32] Build system for python-wrapper Places the wrapper .so file into the Python lib-dynload folder. This should always be in the Path when using Python. The Python Wrapper only builds if necessary build dependencies are found. Signed-off-by: Kevin Vu te Laar --- clients/CMakeLists.txt | 1 + clients/python-wrapper/CMakeLists.txt | 32 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 clients/python-wrapper/CMakeLists.txt diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt index f9ab93112..9ef2a2f11 100644 --- a/clients/CMakeLists.txt +++ b/clients/CMakeLists.txt @@ -5,4 +5,5 @@ # SPDX-License-Identifier: Apache-2.0 add_subdirectory(opal-rt/rtlab-asyncip) +add_subdirectory(python-wrapper) add_subdirectory(shmem) diff --git a/clients/python-wrapper/CMakeLists.txt b/clients/python-wrapper/CMakeLists.txt new file mode 100644 index 000000000..510b7a591 --- /dev/null +++ b/clients/python-wrapper/CMakeLists.txt @@ -0,0 +1,32 @@ +cmake_minimum_required(VERSION 3.15...3.29) +project(villas-python-wrapper LANGUAGES CXX) + +set(PYBIND11_FINDPYTHON ON) +find_package(pybind11 CONFIG OPTIONAL) + +if(pybind11_FOUND) + find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('stdlib') + '/lib-dynload')" + OUTPUT_VARIABLE PYTHON_LIB_DYNLOAD_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + message(STATUS "Found Python version: ${Python_VERSION}") + message(STATUS "Python major version: ${Python_VERSION_MAJOR}") + message(STATUS "Python minor version: ${Python_VERSION_MINOR}") + message(STATUS "Python .so install directory: ${PYTHON_LIB_DYNLOAD_DIR}") + + pybind11_add_module(villas_node villas-python-wrapper.cpp) + target_link_libraries(villas_node PUBLIC villas) + + install( + TARGETS villas_node + COMPONENT lib + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${PYTHON_LIB_DYNLOAD_DIR} + ) +else() + message(STATUS "pybind11 not found. Skipping Python wrapper build.") +endif() From 7302be2c3224263316dfd02e845a5823f0ea1120 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 14 Mar 2025 21:27:54 +0100 Subject: [PATCH 02/32] Added wrapper dependencies to Dockerfiles Added python and pybind11 dependencies to Dockerfiles, except for Fedora minimal, to build the Python-wrapper. Signed-off-by: Kevin Vu te Laar --- packaging/docker/Dockerfile.debian | 5 ++++- packaging/docker/Dockerfile.debian-multiarch | 5 ++++- packaging/docker/Dockerfile.fedora | 3 ++- packaging/docker/Dockerfile.rocky | 6 ++++-- packaging/docker/Dockerfile.ubuntu | 5 ++++- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index b9fcb9801..e88a18116 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -51,7 +51,10 @@ RUN apt-get update && \ liblua5.3-dev \ libhiredis-dev \ libnice-dev \ - libmodbus-dev + libmodbus-dev \ + python3 \ + python3-dev \ + python3-pybind11 # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/packaging/docker/Dockerfile.debian-multiarch b/packaging/docker/Dockerfile.debian-multiarch index 1c07499db..ceb4c4316 100644 --- a/packaging/docker/Dockerfile.debian-multiarch +++ b/packaging/docker/Dockerfile.debian-multiarch @@ -58,7 +58,10 @@ RUN apt-get update && \ libusb-1.0-0-dev:${ARCH} \ liblua5.3-dev:${ARCH} \ libhiredis-dev:${ARCH} \ - libmodbus-dev:${ARCH} + libmodbus-dev:${ARCH} \ + python3:${ARCH} \ + python3-dev:${ARCH} \ + python3-pybind11:${ARCH} # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index d28802f2c..d87ad9e0f 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -64,7 +64,8 @@ RUN dnf -y install \ lua-devel \ hiredis-devel \ libnice-devel \ - libmodbus-devel + libmodbus-devel \ + pybind11-devel # Add local and 64-bit locations to linker paths RUN echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index 96b2cd9f6..a34b8c0f3 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -25,7 +25,8 @@ RUN dnf --allowerasing -y install \ flex bison \ texinfo git curl tar \ protobuf-compiler protobuf-c-compiler \ - clang-tools-extra + clang-tools-extra \ + python python-devel # Dependencies RUN dnf -y install \ @@ -48,7 +49,8 @@ RUN dnf -y install \ lua-devel \ hiredis-devel \ libnice-devel \ - libmodbus-devel + libmodbus-devel \ + pybind11-devel # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index feb4ae2b4..075de2fd8 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -59,7 +59,10 @@ RUN apt-get update && \ libmodbus-dev \ libre2-dev \ libglib2.0-dev \ - libcriterion-dev + libcriterion-dev \ + python3 \ + python3-dev \ + python3-pybind11 # Add local and 64-bit locations to linker paths ENV echo /usr/local/lib >> /etc/ld.so.conf && \ From 3703c20c51aa6cb7f432d0157cc7812436ca5b49 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 14 Mar 2025 21:30:01 +0100 Subject: [PATCH 03/32] Python-Wrapper bindings Added Python-Wrapper bindings. Docstrings are still missing. Signed-off-by: Kevin Vu te Laar --- .../python-wrapper/villas-python-wrapper.cpp | 235 ++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 clients/python-wrapper/villas-python-wrapper.cpp diff --git a/clients/python-wrapper/villas-python-wrapper.cpp b/clients/python-wrapper/villas-python-wrapper.cpp new file mode 100644 index 000000000..78233234b --- /dev/null +++ b/clients/python-wrapper/villas-python-wrapper.cpp @@ -0,0 +1,235 @@ +#include +#include +#include +#include +#include +extern "C" { + #include +} + +namespace py = pybind11; + +class Array { + public: + Array(unsigned int len) { + smps = new vsample *[len](); + this->len = len; + } + ~Array() { + for(unsigned int i = 0; i < len; ++i) { + sample_decref(smps[i]); + smps[i] = nullptr; + } + delete[] smps; + } + + vsample * &operator[](unsigned int idx) { + return smps[idx]; + } + + vsample * &operator[](unsigned int idx) const { + return smps[idx]; + } + + vsample ** get_smps() { + return smps; + } + + unsigned int size() const { + return len; + } + + private: + vsample **smps; + unsigned int len; +}; + +//pybind11 can not deal with (void **) as function input parameters, +//therefore we have to cast a simple (void *) pointer to the corresponding type +// +//wrapper bindings, sorted alphabetically +// @param villas_node Name of the module to be bound +// @param m Access variable for modifying the module code +// +PYBIND11_MODULE(villas_node, m) { + m.def("memory_init", &memory_init); + + m.def("node_check", [](void *n) -> int { + return node_check((vnode *)n); + }); + + m.def("node_destroy", [](void *n) -> int { + return node_destroy((vnode *)n); + }); + + m.def("node_details", [](void *n) -> const char * { + return node_details((vnode *)n); + }, + py::return_value_policy::copy); + + m.def("node_input_signals_max_cnt", [](void *n) -> unsigned { + return node_input_signals_max_cnt((vnode *)n); + }); + + m.def("node_is_enabled", [](void *n) -> bool { + return node_is_enabled((const vnode*)n); + }); + + m.def("node_is_valid_name", [](const char *name) -> bool { + return node_is_valid_name(name); + }); + + m.def("node_name", [](void *n) -> const char * { + return node_name((vnode *)n); + }, + py::return_value_policy::copy); + + m.def("node_name_full", [](void *n) -> const char * { + return node_name_full((vnode *)n); + }, + py::return_value_policy::copy); + + m.def("node_name_short", [](void *n) -> const char * { + return node_name_short((vnode *)n); + }, + py::return_value_policy::copy); + + m.def("node_netem_fds", [](void *n, int fds[]) -> int { + return node_netem_fds((vnode *)n, fds); + }); + + m.def("node_new", [](const char *id_str, const char *json_str) -> vnode * { + json_error_t err; + uuid_t id; + + uuid_parse(id_str, id); + auto *json = json_loads(json_str, 0, &err); + + void *it = json_object_iter(json); + json_t *inner = json_object_iter_value(it); + + if (json_is_object(inner)) { // create node with name + return (vnode *)villas::node::NodeFactory::make(json_object_iter_value(it), id, json_object_iter_key(it)); + } + else { // create node without name + char* capi_str = json_dumps(json, 0); + auto ret = node_new(id_str, capi_str); + + free(capi_str); + return ret; + } + }, + py::return_value_policy::reference); + + m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { + return node_output_signals_max_cnt((vnode *)n); + }); + + m.def("node_pause", [](void *n) -> int { + return node_pause((vnode *)n); + }); + + m.def("node_poll_fds", [](void *n, int fds[]) -> int { + return node_poll_fds((vnode *)n, fds); + }); + + m.def("node_prepare", [](void *n) -> int { + return node_prepare((vsample *)n); + }); + + m.def("node_read", [](void *n, Array &a, unsigned cnt) -> int { + return node_read((vnode *)n, a.get_smps(), cnt); + }); + + m.def("node_restart", [](void *n) -> int { + return node_restart((vnode *)n); + }); + + m.def("node_resume", [](void *n) -> int { + return node_resume((vnode *)n); + }); + + m.def("node_reverse", [](void *n) -> int { + return node_reverse((vnode *)n); + }); + + m.def("node_start", [](void *n) -> int { + return node_start((vnode *)n); + }); + + m.def("node_stop", [](void *n) -> int { + return node_stop((vnode *)n); + }); + + m.def("node_to_json", [](void *n) -> py::str { + auto json = reinterpret_cast(n)->toJson(); + char* json_str = json_dumps(json, 0); + auto py_str = py::str(json_str); + + json_decref(json); + free(json_str); + + return py_str; + }); + + m.def("node_to_json_str", [](void *n) -> py::str { + auto json = reinterpret_cast(n)->toJson(); + char* json_str = json_dumps(json, 0); + auto py_str = py::str(json_str); + + json_decref(json); + free(json_str); + + return py_str; + }); + + m.def("node_write", [](void *n, Array &a, unsigned cnt) -> int { + return node_write((vnode *)n, a.get_smps(), cnt); + }); + + m.def("smps_array", [](unsigned int len) -> Array* { + return new Array(len); + }, + py::return_value_policy::take_ownership); + + m.def("sample_alloc", [](unsigned int len) -> vsample * { + return sample_alloc(len); + }); + + // Decrease reference count and release memory if last reference was held. + m.def("sample_decref", [](void *smps) -> void { + auto smp = (vsample **)smps; + sample_decref(*smp); + }); + + m.def("sample_length", [](void *smp) -> unsigned { + return sample_length((vsample *)smp); + }); + + m.def("sample_pack", &sample_pack, py::return_value_policy::reference); + + m.def("sample_unpack", [](void *smp, unsigned *seq, struct timespec *ts_origin, + struct timespec *ts_received, int *flags, unsigned *len, + double *values) -> void { + return sample_unpack((vsample *)smp, seq, ts_origin, ts_received, flags, len, values); + }, + py::return_value_policy::reference); + + py::class_(m, "SamplesArray") + .def(py::init(), py::arg("len")) + .def("__getitem__", [](Array &a, unsigned int idx) { + if (idx >= a.size()) { + throw py::index_error("Index out of bounds"); + } + return a[idx]; + }) + .def("__setitem__", [](Array &a, unsigned int idx, void *smp) { + if (idx >= a.size()) { + throw py::index_error("Index out of bounds"); + } + if (a[idx]) { + sample_decref(a[idx]); + } + a[idx] = (vsample *)smp; + }); +} From 76ec913c2931db623b22cec2de95a26b0aab3bd9 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 14 Mar 2025 21:30:41 +0100 Subject: [PATCH 04/32] Added Tests for the Python-Wrapper The tests are not yet integrated into the pipeline and are primarily focused on the signal_v2 node. Not all functions work as expected. node_name_short() appears to be print a freed string. node_stop() and node_restart() do not function properly. However, they both use the same underlying function to stop a node, which may be the cause of their failure. Signed-off-by: Kevin Vu te Laar --- tests/unit/python_wrapper.py | 295 +++++++++++++++++++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 tests/unit/python_wrapper.py diff --git a/tests/unit/python_wrapper.py b/tests/unit/python_wrapper.py new file mode 100644 index 000000000..a8b11a3ef --- /dev/null +++ b/tests/unit/python_wrapper.py @@ -0,0 +1,295 @@ +import json +import re +import unittest +import uuid +import villas_node as vn + +class SimpleWrapperTests(unittest.TestCase): + + def setUp(self): + try: + self.node_uuid = str(uuid.uuid4()) + self.config = json.dumps(test_node_config, indent=2) + self.test_node = vn.node_new(self.node_uuid, self.config) + except Exception as e: + self.fail(f"new_node err: {e}") + + def test_activity_changes(self): + try: + vn.node_check(self.test_node) + vn.node_prepare(self.test_node) + # starting twice + self.assertEqual(0, vn.node_start(self.test_node)) + # with self.assertRaises((AssertionError, RuntimeError)): + # vn.node_start(self.test_node) + + # check if the node is running + self.assertTrue(vn.node_is_enabled(self.test_node)) + + # pausing twice + self.assertEqual(0, vn.node_pause(self.test_node)) + self.assertEqual(-1, vn.node_pause(self.test_node)) + + # resuming + self.assertEqual(0, vn.node_resume(self.test_node)) + + # stopping twice + self.assertEqual(0, vn.node_stop(self.test_node)) + # self.assertEqual(-1, vn.node_stop(self.test_node)) + + # # restarting + # vn.node_restart(self.test_node) + + # check if everything still works after restarting + # vn.node_pause(self.test_node) + # vn.node_resume(self.test_node) + # vn.node_stop(self.test_node) + + # terminating the node + vn.node_destroy(self.test_node) + except Exception as e: + self.fail(f" err: {e}") + + def test_details(self): + try: + # remove color codes before checking for equality + self.assertEqual("test_node(socket)", re.sub(r'\x1b\[[0-9;]*m', '', vn.node_name(self.test_node))) + + # node_name_short is bugged + # self.assertEqual('', vn.node_name_short(self.test_node)) + self.assertEqual( + vn.node_name(self.test_node) + + ': uuid=' + + self.node_uuid + + ', #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001', + vn.node_name_full(self.test_node) + ) + + self.assertEqual( + 'layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001', + vn.node_details(self.test_node) + ) + + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f" err: {e}") + + # Test whether or not the json object is created by the wrapper and the module + def test_json_obj(self): + try: + node_config = vn.node_to_json(self.test_node) + node_config_str = json.dumps(node_config) + node_config_parsed = json.loads(node_config_str) + + self.assertEqual(node_config, node_config_parsed) + except Exception as e: + self.fail(f" err: {e}") + + # Test whether or not a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format, thus the name config string will create, as of now, + # a node without a name + # uuid can not match + def test_config_from_string(self): + try: + config_str = vn.node_to_json_str(self.test_node) + config_obj = json.loads(config_str) + + config_copy_str = json.dumps(config_obj, indent=2) + + test_node = vn.node_new("", config_copy_str) + + self.assertEqual( + re.sub(r'^[^:]+: uuid=[0-9a-fA-F-]+, ', '', vn.node_name_full(test_node)), + re.sub(r'^[^:]+: uuid=[0-9a-fA-F-]+, ', '', vn.node_name_full(self.test_node)) + ) + except Exception as e: + self.fail(f" err: {e}") + + def test_rw_socket(self): + try: + data_str = json.dumps(send_recv_test, indent=2) + data = json.loads(data_str) + + test_nodes = {} + for name, content in data.items(): + obj = {name: content} + config = json.dumps(obj, indent=2) + id = str(uuid.uuid4()) + + test_nodes[name] = vn.node_new(id, config) + + for node in test_nodes.values(): + if vn.node_check(node): + raise RuntimeError(f"Failed to verify node configuration") + if vn.node_prepare(node): + raise RuntimeError(f"Failed to verify {vn.node_name(node)} node configuration") + vn.node_start(node) + + # Arrays to store samples + send_smpls = vn.smps_array(1) + intmdt_smpls = vn.smps_array(100) + recv_smpls = vn.smps_array(100) + + for i in range(100): + send_smpls[0] = vn.sample_alloc(2) + intmdt_smpls[i] = vn.sample_alloc(2) + recv_smpls[i] = vn.sample_alloc(2) + + # Generate signals and send over send_socket + self.assertEqual(vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1) + self.assertEqual(vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1) + + # read received signals and send them to recv_socket + self.assertEqual(vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100) + self.assertEqual(vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100) + + # confirm rev_socket signals + self.assertEqual(vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100) + + except Exception as e: + self.fail(f" err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{ + "name": "tap_position", + "type": "integer", + "init": 0 + }] + }, + "out": { + "address": "127.0.0.1:12001" + } + } +} + +send_recv_test = { + "send_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65532", + "signals": [ + { + "name": "voltage", + "type": "float", + "unit": "V" + }, + { + "name": "current", + "type": "float", + "unit": "A" + } + ] + }, + "out": { + "address": "127.0.0.1:65533", + "netem": { + "enabled": False + }, + "multicast": { + "enabled": False + } + } + }, + "intmdt_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65533", + "signals": [ + { + "name": "voltage", + "type": "float", + "unit": "V" + }, + { + "name": "current", + "type": "float", + "unit": "A" + } + ] + }, + "out": { + "address": "127.0.0.1:65534", + "netem": { + "enabled": False + }, + "multicast": { + "enabled": False + } + } + }, + "recv_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65534", + "signals": [ + { + "name": "voltage", + "type": "float", + "unit": "V" + }, + { + "name": "current", + "type": "float", + "unit": "A" + } + ] + }, + "out": { + "address": "127.0.0.1:65535", + "netem": { + "enabled": False + }, + "multicast": { + "enabled": False + } + } + }, + "signal_generator": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V" + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A" + } + ], + "hooks": [ + { + "type": "print", + "format": "villas.human" + } + ] + } + } +} + +if __name__ == '__main__': + unittest.main() From 24abdb7ab4fac47e3a1252804a059ffd42fb8e6b Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 14 Mar 2025 21:41:21 +0100 Subject: [PATCH 05/32] node.cpp Bugfix Fixed logic error. Threw an error if json_t *json is a json object, but it should throw an error if that *json weren't a valid json_object. Signed-off-by: Kevin Vu te Laar --- lib/node.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/node.cpp b/lib/node.cpp index 20b280e62..5c60215b4 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -437,7 +437,7 @@ Node *NodeFactory::make(json_t *json, const uuid_t &id, std::string type; Node *n; - if (json_is_object(json)) + if (!json_is_object(json)) throw ConfigError(json, "node-config-node", "Node configuration must be an object"); From 640e815cf97e8ae5f67787bd95ae13e8ad1d49e1 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 24 Mar 2025 09:24:15 +0100 Subject: [PATCH 06/32] CMake Python-Wrapper test integration, build fix Integrated the Python-Wrapper test "/test/unit/python_wrapper.py" into CMake and the CI. Testing within the Fedora dev container resulted in the issue of missing permissions. Manually running: chmod +x /villas/test/unit/python_wrapper.py helped. Not sure if this issue requires changes for the CI. The build integration with CMake had a little issue. Due to the OPTIONAL flag, pybind11 was never set to found and was therefore never built even though it may have been found by CMake. Changed the target for the python-wrapper build from villas_node to python-wrapper and instead changed its OUTPUT_NAME PROPERTY to villas_node, so that the Python module can still be imported with: "import villas_node". This also avoids confusion with CI and CMake integration. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 1 + clients/python-wrapper/CMakeLists.txt | 9 +++++---- tests/unit/CMakeLists.txt | 10 +++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a310fc4e..12b4b93ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,6 +170,7 @@ test:unit: script: - cmake -S . -B build ${CMAKE_OPTS} - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common + - cmake --build build ${CMAKE_BUILD_OPTS} --target python-wrapper-tests needs: - job: "build:source: [fedora]" artifacts: true diff --git a/clients/python-wrapper/CMakeLists.txt b/clients/python-wrapper/CMakeLists.txt index 510b7a591..c25546a9a 100644 --- a/clients/python-wrapper/CMakeLists.txt +++ b/clients/python-wrapper/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.15...3.29) project(villas-python-wrapper LANGUAGES CXX) set(PYBIND11_FINDPYTHON ON) -find_package(pybind11 CONFIG OPTIONAL) +find_package(pybind11 CONFIG) if(pybind11_FOUND) find_package(Python3 COMPONENTS Interpreter Development REQUIRED) @@ -18,11 +18,12 @@ if(pybind11_FOUND) message(STATUS "Python minor version: ${Python_VERSION_MINOR}") message(STATUS "Python .so install directory: ${PYTHON_LIB_DYNLOAD_DIR}") - pybind11_add_module(villas_node villas-python-wrapper.cpp) - target_link_libraries(villas_node PUBLIC villas) + pybind11_add_module(python-wrapper villas-python-wrapper.cpp) + set_target_properties(python-wrapper PROPERTIES OUTPUT_NAME villas_node) + target_link_libraries(python-wrapper PUBLIC villas) install( - TARGETS villas_node + TARGETS python-wrapper COMPONENT lib RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${PYTHON_LIB_DYNLOAD_DIR} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f9098e53e..035ea3f23 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,5 +35,13 @@ add_custom_target(run-unit-tests USES_TERMINAL ) +add_custom_target(run-python-wrapper-tests + COMMAND + ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/unit/python_wrapper.py + DEPENDS + python-wrapper + USES_TERMINAL +) + add_dependencies(tests unit-tests) -add_dependencies(run-tests run-unit-tests) +add_dependencies(run-tests run-unit-tests run-python-wrapper-tests) From 862d498c4d0ee2cb2aa39dd09c97780c0185d8f2 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 24 Mar 2025 10:15:27 +0100 Subject: [PATCH 07/32] Applied changes as per feedback. Signed-off-by: Kevin Vu te Laar --- clients/python-wrapper/CMakeLists.txt | 6 +++++ .../python-wrapper/villas-python-wrapper.cpp | 22 +++++++++++++------ packaging/docker/Dockerfile.rocky | 2 +- tests/unit/python_wrapper.py | 6 +++++ 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/clients/python-wrapper/CMakeLists.txt b/clients/python-wrapper/CMakeLists.txt index c25546a9a..d890dc345 100644 --- a/clients/python-wrapper/CMakeLists.txt +++ b/clients/python-wrapper/CMakeLists.txt @@ -1,3 +1,9 @@ +# Python-wrapper CMakeLists. +# +# Author: Kevin Vu te Laar +# SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + cmake_minimum_required(VERSION 3.15...3.29) project(villas-python-wrapper LANGUAGES CXX) diff --git a/clients/python-wrapper/villas-python-wrapper.cpp b/clients/python-wrapper/villas-python-wrapper.cpp index 78233234b..dab2dc8f2 100644 --- a/clients/python-wrapper/villas-python-wrapper.cpp +++ b/clients/python-wrapper/villas-python-wrapper.cpp @@ -1,8 +1,16 @@ +/* Python-wrapper. + * + * Author: Kevin Vu te Laar + * SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + #include #include #include #include #include + extern "C" { #include } @@ -44,13 +52,13 @@ class Array { unsigned int len; }; -//pybind11 can not deal with (void **) as function input parameters, -//therefore we have to cast a simple (void *) pointer to the corresponding type -// -//wrapper bindings, sorted alphabetically -// @param villas_node Name of the module to be bound -// @param m Access variable for modifying the module code -// +/* pybind11 can not deal with (void **) as function input parameters, + * therefore we have to cast a simple (void *) pointer to the corresponding type + * + * wrapper bindings, sorted alphabetically + * @param villas_node Name of the module to be bound + * @param m Access variable for modifying the module code + */ PYBIND11_MODULE(villas_node, m) { m.def("memory_init", &memory_init); diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index a34b8c0f3..e14abd0b4 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -26,7 +26,7 @@ RUN dnf --allowerasing -y install \ texinfo git curl tar \ protobuf-compiler protobuf-c-compiler \ clang-tools-extra \ - python python-devel + python python-devel # Dependencies RUN dnf -y install \ diff --git a/tests/unit/python_wrapper.py b/tests/unit/python_wrapper.py index a8b11a3ef..3c6437170 100644 --- a/tests/unit/python_wrapper.py +++ b/tests/unit/python_wrapper.py @@ -1,3 +1,9 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + import json import re import unittest From 73ce98f6b3ecea2838cbe30c14457e3aedd3cddf Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 8 Apr 2025 09:10:46 +0200 Subject: [PATCH 08/32] Fixed typo The target is `run-python-wrapper-tests` instead of `python-wrapper-tests` Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12b4b93ce..b7426c6be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -170,7 +170,7 @@ test:unit: script: - cmake -S . -B build ${CMAKE_OPTS} - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common - - cmake --build build ${CMAKE_BUILD_OPTS} --target python-wrapper-tests + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-wrapper-tests needs: - job: "build:source: [fedora]" artifacts: true From 168df2103fcbfb914734040bbc3ffc7739cc6403 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 8 Apr 2025 09:11:26 +0200 Subject: [PATCH 09/32] Explicitly disallowed copy and assignment cppcheck threw a warning during the pipeline. Since copy and assignment are not used and should not be used, this is fine and won't result in any difference. Calling .copy() in python would have also thrown an error earlier, as a copyconstructor wasn't explicitly exposed to pybind via bindings. Signed-off-by: Kevin Vu te Laar --- clients/python-wrapper/villas-python-wrapper.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clients/python-wrapper/villas-python-wrapper.cpp b/clients/python-wrapper/villas-python-wrapper.cpp index dab2dc8f2..d0d20ddbe 100644 --- a/clients/python-wrapper/villas-python-wrapper.cpp +++ b/clients/python-wrapper/villas-python-wrapper.cpp @@ -23,6 +23,9 @@ class Array { smps = new vsample *[len](); this->len = len; } + Array(const Array&) = delete; + Array& operator= (const Array&) = delete; + ~Array() { for(unsigned int i = 0; i < len; ++i) { sample_decref(smps[i]); From e710de5667e63dba388744c7923bec959d49eb4a Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 8 Apr 2025 10:39:47 +0200 Subject: [PATCH 10/32] CMakeLists.txt fix Use find_package(Python3 REQUIRED) to set Python3_EXECUTABLE. Moved the `REQUIRED` to align with CMake style guidelines. Signed-off-by: Kevin Vu te Laar --- clients/python-wrapper/CMakeLists.txt | 2 +- tests/unit/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/python-wrapper/CMakeLists.txt b/clients/python-wrapper/CMakeLists.txt index d890dc345..b7ad31f13 100644 --- a/clients/python-wrapper/CMakeLists.txt +++ b/clients/python-wrapper/CMakeLists.txt @@ -11,7 +11,7 @@ set(PYBIND11_FINDPYTHON ON) find_package(pybind11 CONFIG) if(pybind11_FOUND) - find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + find_package(Python3 REQUIRED COMPONENTS Interpreter Development) execute_process( COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('stdlib') + '/lib-dynload')" diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 035ea3f23..7aced7ef0 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,6 +35,7 @@ add_custom_target(run-unit-tests USES_TERMINAL ) +find_package(Python3 REQUIRED COMPONENTS Interpreter) add_custom_target(run-python-wrapper-tests COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/unit/python_wrapper.py From b1e9a2f637f3430af9c9f2cb44ef3fdbbfdbac54 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 15 May 2025 01:59:07 +0200 Subject: [PATCH 11/32] Removed unnecessary dockerfile dependencies Based on suggestions from PR#884. Removed the `python` dependency because `python-devel` in Debian and Rocky also depend on the base `python` package. Signed-off-by: Kevin Vu te Laar --- packaging/docker/Dockerfile.debian | 1 - packaging/docker/Dockerfile.debian-multiarch | 1 - packaging/docker/Dockerfile.fedora | 2 +- packaging/docker/Dockerfile.rocky | 2 +- packaging/docker/Dockerfile.ubuntu | 1 - 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index e88a18116..a5881d234 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -52,7 +52,6 @@ RUN apt-get update && \ libhiredis-dev \ libnice-dev \ libmodbus-dev \ - python3 \ python3-dev \ python3-pybind11 diff --git a/packaging/docker/Dockerfile.debian-multiarch b/packaging/docker/Dockerfile.debian-multiarch index ceb4c4316..23c466e80 100644 --- a/packaging/docker/Dockerfile.debian-multiarch +++ b/packaging/docker/Dockerfile.debian-multiarch @@ -59,7 +59,6 @@ RUN apt-get update && \ liblua5.3-dev:${ARCH} \ libhiredis-dev:${ARCH} \ libmodbus-dev:${ARCH} \ - python3:${ARCH} \ python3-dev:${ARCH} \ python3-pybind11:${ARCH} diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index f7f487bca..36a6d2034 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -27,7 +27,7 @@ RUN dnf -y install \ openssh-clients \ jq nmap-ncat \ iproute iproute-tc \ - python python-devel python-pip \ + python-devel python-pip \ gdb gdb-gdbserver \ cppcheck \ xmlto dblatex rubygem-asciidoctor \ diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index 067cc1e66..760a307b1 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -26,7 +26,7 @@ RUN dnf --allowerasing -y install \ texinfo git curl tar \ protobuf-compiler protobuf-c-compiler \ clang-tools-extra \ - python python-devel + python-devel # Dependencies RUN dnf -y install \ diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index 075de2fd8..200226e31 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -60,7 +60,6 @@ RUN apt-get update && \ libre2-dev \ libglib2.0-dev \ libcriterion-dev \ - python3 \ python3-dev \ python3-pybind11 From 365f88b508e8d74c9ddf32dc856689060aee62ba Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 15 May 2025 01:33:33 +0200 Subject: [PATCH 12/32] Renamed wrapper -> binding, LLVM format As suggested `wrapper` was substituted by `binding`. The minimum required version and project naming in `CMakeLists.txt` were removed, as this is not necessary. Signed-off-by: Kevin Vu te Laar --- clients/CMakeLists.txt | 2 +- .../CMakeLists.txt | 13 +- .../python-binding/villas-python-binding.cpp | 226 ++++++++++++++++ .../python-wrapper/villas-python-wrapper.cpp | 246 ------------------ .../{python_wrapper.py => python_binding.py} | 0 5 files changed, 232 insertions(+), 255 deletions(-) rename clients/{python-wrapper => python-binding}/CMakeLists.txt (76%) create mode 100644 clients/python-binding/villas-python-binding.cpp delete mode 100644 clients/python-wrapper/villas-python-wrapper.cpp rename tests/unit/{python_wrapper.py => python_binding.py} (100%) diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt index 9ef2a2f11..a5cf995ef 100644 --- a/clients/CMakeLists.txt +++ b/clients/CMakeLists.txt @@ -5,5 +5,5 @@ # SPDX-License-Identifier: Apache-2.0 add_subdirectory(opal-rt/rtlab-asyncip) -add_subdirectory(python-wrapper) +add_subdirectory(python-binding) add_subdirectory(shmem) diff --git a/clients/python-wrapper/CMakeLists.txt b/clients/python-binding/CMakeLists.txt similarity index 76% rename from clients/python-wrapper/CMakeLists.txt rename to clients/python-binding/CMakeLists.txt index b7ad31f13..fd645f338 100644 --- a/clients/python-wrapper/CMakeLists.txt +++ b/clients/python-binding/CMakeLists.txt @@ -1,12 +1,9 @@ -# Python-wrapper CMakeLists. +# Python-binding CMakeLists. # # Author: Kevin Vu te Laar # SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.15...3.29) -project(villas-python-wrapper LANGUAGES CXX) - set(PYBIND11_FINDPYTHON ON) find_package(pybind11 CONFIG) @@ -24,12 +21,12 @@ if(pybind11_FOUND) message(STATUS "Python minor version: ${Python_VERSION_MINOR}") message(STATUS "Python .so install directory: ${PYTHON_LIB_DYNLOAD_DIR}") - pybind11_add_module(python-wrapper villas-python-wrapper.cpp) - set_target_properties(python-wrapper PROPERTIES OUTPUT_NAME villas_node) - target_link_libraries(python-wrapper PUBLIC villas) + pybind11_add_module(python-binding villas-python-binding.cpp) + set_target_properties(python-binding PROPERTIES OUTPUT_NAME villas_node) + target_link_libraries(python-binding PUBLIC villas) install( - TARGETS python-wrapper + TARGETS python-binding COMPONENT lib RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${PYTHON_LIB_DYNLOAD_DIR} diff --git a/clients/python-binding/villas-python-binding.cpp b/clients/python-binding/villas-python-binding.cpp new file mode 100644 index 000000000..ff97d2ab9 --- /dev/null +++ b/clients/python-binding/villas-python-binding.cpp @@ -0,0 +1,226 @@ +/* Python-binding. + * + * Author: Kevin Vu te Laar + * SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power + * Systems, RWTH Aachen University SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include + +extern "C" { +#include +} + +namespace py = pybind11; + +class Array { +public: + Array(unsigned int len) { + smps = new vsample *[len](); + this->len = len; + } + Array(const Array &) = delete; + Array &operator=(const Array &) = delete; + + ~Array() { + for (unsigned int i = 0; i < len; ++i) { + sample_decref(smps[i]); + smps[i] = nullptr; + } + delete[] smps; + } + + vsample *&operator[](unsigned int idx) { return smps[idx]; } + + vsample *&operator[](unsigned int idx) const { return smps[idx]; } + + vsample **get_smps() { return smps; } + + unsigned int size() const { return len; } + +private: + vsample **smps; + unsigned int len; +}; + +/* pybind11 can not deal with (void **) as function input parameters, + * therefore we have to cast a simple (void *) pointer to the corresponding type + * + * wrapper bindings, sorted alphabetically + * @param villas_node Name of the module to be bound + * @param m Access variable for modifying the module code + */ +PYBIND11_MODULE(villas_node, m) { + m.def("memory_init", &memory_init); + + m.def("node_check", [](void *n) -> int { return node_check((vnode *)n); }); + + m.def("node_destroy", + [](void *n) -> int { return node_destroy((vnode *)n); }); + + m.def( + "node_details", + [](void *n) -> const char * { return node_details((vnode *)n); }, + py::return_value_policy::copy); + + m.def("node_input_signals_max_cnt", [](void *n) -> unsigned { + return node_input_signals_max_cnt((vnode *)n); + }); + + m.def("node_is_enabled", + [](void *n) -> bool { return node_is_enabled((const vnode *)n); }); + + m.def("node_is_valid_name", + [](const char *name) -> bool { return node_is_valid_name(name); }); + + m.def( + "node_name", + [](void *n) -> const char * { return node_name((vnode *)n); }, + py::return_value_policy::copy); + + m.def( + "node_name_full", + [](void *n) -> const char * { return node_name_full((vnode *)n); }, + py::return_value_policy::copy); + + m.def( + "node_name_short", + [](void *n) -> const char * { return node_name_short((vnode *)n); }, + py::return_value_policy::copy); + + m.def("node_netem_fds", [](void *n, int fds[]) -> int { + return node_netem_fds((vnode *)n, fds); + }); + + m.def( + "node_new", + [](const char *id_str, const char *json_str) -> vnode * { + json_error_t err; + uuid_t id; + + uuid_parse(id_str, id); + auto *json = json_loads(json_str, 0, &err); + + void *it = json_object_iter(json); + json_t *inner = json_object_iter_value(it); + + if (json_is_object(inner)) { // create node with name + return (vnode *)villas::node::NodeFactory::make( + json_object_iter_value(it), id, json_object_iter_key(it)); + } else { // create node without name + char *capi_str = json_dumps(json, 0); + auto ret = node_new(id_str, capi_str); + + free(capi_str); + return ret; + } + }, + py::return_value_policy::reference); + + m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { + return node_output_signals_max_cnt((vnode *)n); + }); + + m.def("node_pause", [](void *n) -> int { return node_pause((vnode *)n); }); + + m.def("node_poll_fds", [](void *n, int fds[]) -> int { + return node_poll_fds((vnode *)n, fds); + }); + + m.def("node_prepare", + [](void *n) -> int { return node_prepare((vsample *)n); }); + + m.def("node_read", [](void *n, Array &a, unsigned cnt) -> int { + return node_read((vnode *)n, a.get_smps(), cnt); + }); + + m.def("node_restart", + [](void *n) -> int { return node_restart((vnode *)n); }); + + m.def("node_resume", [](void *n) -> int { return node_resume((vnode *)n); }); + + m.def("node_reverse", + [](void *n) -> int { return node_reverse((vnode *)n); }); + + m.def("node_start", [](void *n) -> int { return node_start((vnode *)n); }); + + m.def("node_stop", [](void *n) -> int { return node_stop((vnode *)n); }); + + m.def("node_to_json", [](void *n) -> py::str { + auto json = reinterpret_cast(n)->toJson(); + char *json_str = json_dumps(json, 0); + auto py_str = py::str(json_str); + + json_decref(json); + free(json_str); + + return py_str; + }); + + m.def("node_to_json_str", [](void *n) -> py::str { + auto json = reinterpret_cast(n)->toJson(); + char *json_str = json_dumps(json, 0); + auto py_str = py::str(json_str); + + json_decref(json); + free(json_str); + + return py_str; + }); + + m.def("node_write", [](void *n, Array &a, unsigned cnt) -> int { + return node_write((vnode *)n, a.get_smps(), cnt); + }); + + m.def( + "smps_array", [](unsigned int len) -> Array * { return new Array(len); }, + py::return_value_policy::take_ownership); + + m.def("sample_alloc", + [](unsigned int len) -> vsample * { return sample_alloc(len); }); + + // Decrease reference count and release memory if last reference was held. + m.def("sample_decref", [](void *smps) -> void { + auto smp = (vsample **)smps; + sample_decref(*smp); + }); + + m.def("sample_length", + [](void *smp) -> unsigned { return sample_length((vsample *)smp); }); + + m.def("sample_pack", &sample_pack, py::return_value_policy::reference); + + m.def( + "sample_unpack", + [](void *smp, unsigned *seq, struct timespec *ts_origin, + struct timespec *ts_received, int *flags, unsigned *len, + double *values) -> void { + return sample_unpack((vsample *)smp, seq, ts_origin, ts_received, flags, + len, values); + }, + py::return_value_policy::reference); + + py::class_(m, "SamplesArray") + .def(py::init(), py::arg("len")) + .def("__getitem__", + [](Array &a, unsigned int idx) { + if (idx >= a.size()) { + throw py::index_error("Index out of bounds"); + } + return a[idx]; + }) + .def("__setitem__", [](Array &a, unsigned int idx, void *smp) { + if (idx >= a.size()) { + throw py::index_error("Index out of bounds"); + } + if (a[idx]) { + sample_decref(a[idx]); + } + a[idx] = (vsample *)smp; + }); +} diff --git a/clients/python-wrapper/villas-python-wrapper.cpp b/clients/python-wrapper/villas-python-wrapper.cpp deleted file mode 100644 index d0d20ddbe..000000000 --- a/clients/python-wrapper/villas-python-wrapper.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/* Python-wrapper. - * - * Author: Kevin Vu te Laar - * SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include - -extern "C" { - #include -} - -namespace py = pybind11; - -class Array { - public: - Array(unsigned int len) { - smps = new vsample *[len](); - this->len = len; - } - Array(const Array&) = delete; - Array& operator= (const Array&) = delete; - - ~Array() { - for(unsigned int i = 0; i < len; ++i) { - sample_decref(smps[i]); - smps[i] = nullptr; - } - delete[] smps; - } - - vsample * &operator[](unsigned int idx) { - return smps[idx]; - } - - vsample * &operator[](unsigned int idx) const { - return smps[idx]; - } - - vsample ** get_smps() { - return smps; - } - - unsigned int size() const { - return len; - } - - private: - vsample **smps; - unsigned int len; -}; - -/* pybind11 can not deal with (void **) as function input parameters, - * therefore we have to cast a simple (void *) pointer to the corresponding type - * - * wrapper bindings, sorted alphabetically - * @param villas_node Name of the module to be bound - * @param m Access variable for modifying the module code - */ -PYBIND11_MODULE(villas_node, m) { - m.def("memory_init", &memory_init); - - m.def("node_check", [](void *n) -> int { - return node_check((vnode *)n); - }); - - m.def("node_destroy", [](void *n) -> int { - return node_destroy((vnode *)n); - }); - - m.def("node_details", [](void *n) -> const char * { - return node_details((vnode *)n); - }, - py::return_value_policy::copy); - - m.def("node_input_signals_max_cnt", [](void *n) -> unsigned { - return node_input_signals_max_cnt((vnode *)n); - }); - - m.def("node_is_enabled", [](void *n) -> bool { - return node_is_enabled((const vnode*)n); - }); - - m.def("node_is_valid_name", [](const char *name) -> bool { - return node_is_valid_name(name); - }); - - m.def("node_name", [](void *n) -> const char * { - return node_name((vnode *)n); - }, - py::return_value_policy::copy); - - m.def("node_name_full", [](void *n) -> const char * { - return node_name_full((vnode *)n); - }, - py::return_value_policy::copy); - - m.def("node_name_short", [](void *n) -> const char * { - return node_name_short((vnode *)n); - }, - py::return_value_policy::copy); - - m.def("node_netem_fds", [](void *n, int fds[]) -> int { - return node_netem_fds((vnode *)n, fds); - }); - - m.def("node_new", [](const char *id_str, const char *json_str) -> vnode * { - json_error_t err; - uuid_t id; - - uuid_parse(id_str, id); - auto *json = json_loads(json_str, 0, &err); - - void *it = json_object_iter(json); - json_t *inner = json_object_iter_value(it); - - if (json_is_object(inner)) { // create node with name - return (vnode *)villas::node::NodeFactory::make(json_object_iter_value(it), id, json_object_iter_key(it)); - } - else { // create node without name - char* capi_str = json_dumps(json, 0); - auto ret = node_new(id_str, capi_str); - - free(capi_str); - return ret; - } - }, - py::return_value_policy::reference); - - m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { - return node_output_signals_max_cnt((vnode *)n); - }); - - m.def("node_pause", [](void *n) -> int { - return node_pause((vnode *)n); - }); - - m.def("node_poll_fds", [](void *n, int fds[]) -> int { - return node_poll_fds((vnode *)n, fds); - }); - - m.def("node_prepare", [](void *n) -> int { - return node_prepare((vsample *)n); - }); - - m.def("node_read", [](void *n, Array &a, unsigned cnt) -> int { - return node_read((vnode *)n, a.get_smps(), cnt); - }); - - m.def("node_restart", [](void *n) -> int { - return node_restart((vnode *)n); - }); - - m.def("node_resume", [](void *n) -> int { - return node_resume((vnode *)n); - }); - - m.def("node_reverse", [](void *n) -> int { - return node_reverse((vnode *)n); - }); - - m.def("node_start", [](void *n) -> int { - return node_start((vnode *)n); - }); - - m.def("node_stop", [](void *n) -> int { - return node_stop((vnode *)n); - }); - - m.def("node_to_json", [](void *n) -> py::str { - auto json = reinterpret_cast(n)->toJson(); - char* json_str = json_dumps(json, 0); - auto py_str = py::str(json_str); - - json_decref(json); - free(json_str); - - return py_str; - }); - - m.def("node_to_json_str", [](void *n) -> py::str { - auto json = reinterpret_cast(n)->toJson(); - char* json_str = json_dumps(json, 0); - auto py_str = py::str(json_str); - - json_decref(json); - free(json_str); - - return py_str; - }); - - m.def("node_write", [](void *n, Array &a, unsigned cnt) -> int { - return node_write((vnode *)n, a.get_smps(), cnt); - }); - - m.def("smps_array", [](unsigned int len) -> Array* { - return new Array(len); - }, - py::return_value_policy::take_ownership); - - m.def("sample_alloc", [](unsigned int len) -> vsample * { - return sample_alloc(len); - }); - - // Decrease reference count and release memory if last reference was held. - m.def("sample_decref", [](void *smps) -> void { - auto smp = (vsample **)smps; - sample_decref(*smp); - }); - - m.def("sample_length", [](void *smp) -> unsigned { - return sample_length((vsample *)smp); - }); - - m.def("sample_pack", &sample_pack, py::return_value_policy::reference); - - m.def("sample_unpack", [](void *smp, unsigned *seq, struct timespec *ts_origin, - struct timespec *ts_received, int *flags, unsigned *len, - double *values) -> void { - return sample_unpack((vsample *)smp, seq, ts_origin, ts_received, flags, len, values); - }, - py::return_value_policy::reference); - - py::class_(m, "SamplesArray") - .def(py::init(), py::arg("len")) - .def("__getitem__", [](Array &a, unsigned int idx) { - if (idx >= a.size()) { - throw py::index_error("Index out of bounds"); - } - return a[idx]; - }) - .def("__setitem__", [](Array &a, unsigned int idx, void *smp) { - if (idx >= a.size()) { - throw py::index_error("Index out of bounds"); - } - if (a[idx]) { - sample_decref(a[idx]); - } - a[idx] = (vsample *)smp; - }); -} diff --git a/tests/unit/python_wrapper.py b/tests/unit/python_binding.py similarity index 100% rename from tests/unit/python_wrapper.py rename to tests/unit/python_binding.py From 6806d2409f5fba50dbed4b01589b20e518bf5787 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 15 May 2025 01:37:47 +0200 Subject: [PATCH 13/32] Changes for pipeline test integration - Removed unnecessary `find_package()` call in `CMakeLists.txt`. - Changed naming to match substitution of `wrapper` with `binding`. - Bumped the version requirement from `3.14` to `3.15` matching the removed minimum in `binding` `CMakeLists.txt`. - Dropped version range syntax `cmake_minimum_required(A...B)`, introduced in `3.19`, in favor of a simple minimum requirement of `3.15`. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 4 ++-- CMakeLists.txt | 2 +- tests/unit/CMakeLists.txt | 9 ++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7426c6be..40358515c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -169,8 +169,8 @@ test:unit: image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-wrapper-tests + - export PYTHONPATH=$PYTHONPATH:${PWD}/build/clients/python-binding + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-binding-tests needs: - job: "build:source: [fedora]" artifacts: true diff --git a/CMakeLists.txt b/CMakeLists.txt index ba316a1b7..738cafe20 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.14) +cmake_minimum_required(VERSION 3.15) project(villas-node VERSION 0.0.0 diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 7aced7ef0..3a1394cb1 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,14 +35,13 @@ add_custom_target(run-unit-tests USES_TERMINAL ) -find_package(Python3 REQUIRED COMPONENTS Interpreter) -add_custom_target(run-python-wrapper-tests +add_custom_target(run-python-binding-tests COMMAND - ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/unit/python_wrapper.py + python3 ${CMAKE_SOURCE_DIR}/tests/unit/python_binding.py DEPENDS - python-wrapper + python-binding USES_TERMINAL ) add_dependencies(tests unit-tests) -add_dependencies(run-tests run-unit-tests run-python-wrapper-tests) +add_dependencies(run-tests run-unit-tests run-python-binding-tests) From 92163665db9fc53cb260e75dae2a35f4a5f3065a Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 15 May 2025 12:57:49 +0200 Subject: [PATCH 14/32] Binding test formatted with black Manually formatted the Python binding test with black. The pipeline formatting resulted in an invalid escape sequence `\m`. Oddly enough there is no such character sequence within the code. Especially not at the reported line 108 when looking at: https://git.rwth-aachen.de/acs/public/villas/node/-/jobs/6248036 . Signed-off-by: Kevin Vu te Laar --- tests/unit/python_binding.py | 157 +++++++++++++++-------------------- 1 file changed, 66 insertions(+), 91 deletions(-) diff --git a/tests/unit/python_binding.py b/tests/unit/python_binding.py index 3c6437170..b3b10355f 100644 --- a/tests/unit/python_binding.py +++ b/tests/unit/python_binding.py @@ -10,6 +10,7 @@ import uuid import villas_node as vn + class SimpleWrapperTests(unittest.TestCase): def setUp(self): @@ -59,21 +60,24 @@ def test_activity_changes(self): def test_details(self): try: # remove color codes before checking for equality - self.assertEqual("test_node(socket)", re.sub(r'\x1b\[[0-9;]*m', '', vn.node_name(self.test_node))) + self.assertEqual( + "test_node(socket)", + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name(self.test_node)), + ) # node_name_short is bugged # self.assertEqual('', vn.node_name_short(self.test_node)) self.assertEqual( - vn.node_name(self.test_node) + - ': uuid=' + - self.node_uuid + - ', #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001', - vn.node_name_full(self.test_node) + vn.node_name(self.test_node) + + ": uuid=" + + self.node_uuid + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + vn.node_name_full(self.test_node), ) self.assertEqual( - 'layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001', - vn.node_details(self.test_node) + "layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + vn.node_details(self.test_node), ) self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) @@ -106,8 +110,14 @@ def test_config_from_string(self): test_node = vn.node_new("", config_copy_str) self.assertEqual( - re.sub(r'^[^:]+: uuid=[0-9a-fA-F-]+, ', '', vn.node_name_full(test_node)), - re.sub(r'^[^:]+: uuid=[0-9a-fA-F-]+, ', '', vn.node_name_full(self.test_node)) + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) + ), + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + vn.node_name_full(self.test_node), + ), ) except Exception as e: self.fail(f" err: {e}") @@ -127,9 +137,11 @@ def test_rw_socket(self): for node in test_nodes.values(): if vn.node_check(node): - raise RuntimeError(f"Failed to verify node configuration") + raise RuntimeError("Failed to verify node configuration") if vn.node_prepare(node): - raise RuntimeError(f"Failed to verify {vn.node_name(node)} node configuration") + raise RuntimeError( + f"Failed to verify {vn.node_name(node)} node configuration" + ) vn.node_start(node) # Arrays to store samples @@ -143,15 +155,25 @@ def test_rw_socket(self): recv_smpls[i] = vn.sample_alloc(2) # Generate signals and send over send_socket - self.assertEqual(vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1) - self.assertEqual(vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1) + self.assertEqual( + vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1 + ) + self.assertEqual( + vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + ) # read received signals and send them to recv_socket - self.assertEqual(vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100) - self.assertEqual(vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100) + self.assertEqual( + vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + ) + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + ) # confirm rev_socket signals - self.assertEqual(vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100) + self.assertEqual( + vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) except Exception as e: self.fail(f" err: {e}") @@ -164,15 +186,9 @@ def test_rw_socket(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [{ - "name": "tap_position", - "type": "integer", - "init": 0 - }] + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], }, - "out": { - "address": "127.0.0.1:12001" - } + "out": {"address": "127.0.0.1:12001"}, } } @@ -184,27 +200,15 @@ def test_rw_socket(self): "in": { "address": "127.0.0.1:65532", "signals": [ - { - "name": "voltage", - "type": "float", - "unit": "V" - }, - { - "name": "current", - "type": "float", - "unit": "A" - } - ] + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], }, "out": { "address": "127.0.0.1:65533", - "netem": { - "enabled": False - }, - "multicast": { - "enabled": False - } - } + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, }, "intmdt_socket": { "type": "socket", @@ -213,27 +217,15 @@ def test_rw_socket(self): "in": { "address": "127.0.0.1:65533", "signals": [ - { - "name": "voltage", - "type": "float", - "unit": "V" - }, - { - "name": "current", - "type": "float", - "unit": "A" - } - ] + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], }, "out": { "address": "127.0.0.1:65534", - "netem": { - "enabled": False - }, - "multicast": { - "enabled": False - } - } + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, }, "recv_socket": { "type": "socket", @@ -242,27 +234,15 @@ def test_rw_socket(self): "in": { "address": "127.0.0.1:65534", "signals": [ - { - "name": "voltage", - "type": "float", - "unit": "V" - }, - { - "name": "current", - "type": "float", - "unit": "A" - } - ] + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], }, "out": { "address": "127.0.0.1:65535", - "netem": { - "enabled": False - }, - "multicast": { - "enabled": False - } - } + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, }, "signal_generator": { "type": "signal.v2", @@ -276,7 +256,7 @@ def test_rw_socket(self): "phase": 90, "signal": "sine", "type": "float", - "unit": "V" + "unit": "V", }, { "amplitude": 1, @@ -284,18 +264,13 @@ def test_rw_socket(self): "phase": 0, "signal": "sine", "type": "float", - "unit": "A" - } + "unit": "A", + }, ], - "hooks": [ - { - "type": "print", - "format": "villas.human" - } - ] - } - } + "hooks": [{"type": "print", "format": "villas.human"}], + }, + }, } -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() From 2d7628f539b221ec746d6878fe2500d017e25ecd Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 22 May 2025 14:16:31 +0200 Subject: [PATCH 15/32] Format Python binding test with black-jupyter Manually formatted the Python binding test with black-jupyter. The pipeline formatting resulted in an "invalid escape sequence \m" error. Oddly enough there is no such character sequence within the code. Especially not at the reported line 108 when considering: https://git.rwth-aachen.de/acs/public/villas/node/-/jobs/6248036 https://git.rwth-aachen.de/acs/public/villas/node/-/jobs/6249567 `black-jupyter` seems to enforce stricter formatting rules than plain `black`. Formatting with plain `black` caused the pipeline to fail, as `black-jupyter` made additional changes. Signed-off-by: Kevin Vu te Laar --- tests/unit/python_binding.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/python_binding.py b/tests/unit/python_binding.py index b3b10355f..f528873cc 100644 --- a/tests/unit/python_binding.py +++ b/tests/unit/python_binding.py @@ -12,7 +12,6 @@ class SimpleWrapperTests(unittest.TestCase): - def setUp(self): try: self.node_uuid = str(uuid.uuid4()) From 07cffec1551534c940128f9e2091801e4342b0e9 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 12 Jun 2025 11:30:34 +0200 Subject: [PATCH 16/32] Align python-devel package for consistency Moved `python-devel` to align with the Ubuntu and Debian Dockerfile. The Fedora one remains the exception. Signed-off-by: Kevin Vu te Laar --- packaging/docker/Dockerfile.rocky | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index f9c049fbf..8619be8a5 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -25,8 +25,7 @@ RUN dnf --allowerasing -y install \ flex bison \ texinfo git git-svn curl tar patchutils \ protobuf-compiler protobuf-c-compiler \ - clang-tools-extra \ - python-devel + clang-tools-extra # Dependencies RUN dnf -y install \ @@ -50,6 +49,7 @@ RUN dnf -y install \ hiredis-devel \ libnice-devel \ libmodbus-devel \ + python-devel \ pybind11-devel # Install unpackaged dependencies from source From d9d12dedfcdc799006fa7b40f50a11d63230a468 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 17 Jul 2025 16:34:33 +0200 Subject: [PATCH 17/32] Moved clients/python-binding/ to python/binding/ Signed-off-by: Kevin Vu te Laar --- clients/CMakeLists.txt | 1 - python/CMakeLists.txt | 2 ++ {clients/python-binding => python/binding}/CMakeLists.txt | 0 .../python-binding => python/binding}/villas-python-binding.cpp | 0 4 files changed, 2 insertions(+), 1 deletion(-) rename {clients/python-binding => python/binding}/CMakeLists.txt (100%) rename {clients/python-binding => python/binding}/villas-python-binding.cpp (100%) diff --git a/clients/CMakeLists.txt b/clients/CMakeLists.txt index a5cf995ef..f9ab93112 100644 --- a/clients/CMakeLists.txt +++ b/clients/CMakeLists.txt @@ -5,5 +5,4 @@ # SPDX-License-Identifier: Apache-2.0 add_subdirectory(opal-rt/rtlab-asyncip) -add_subdirectory(python-binding) add_subdirectory(shmem) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f9aa07f2f..e0f32dbc9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,6 +4,8 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 +add_subdirectory(binding) + if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND) add_custom_command( OUTPUT diff --git a/clients/python-binding/CMakeLists.txt b/python/binding/CMakeLists.txt similarity index 100% rename from clients/python-binding/CMakeLists.txt rename to python/binding/CMakeLists.txt diff --git a/clients/python-binding/villas-python-binding.cpp b/python/binding/villas-python-binding.cpp similarity index 100% rename from clients/python-binding/villas-python-binding.cpp rename to python/binding/villas-python-binding.cpp From a146ecbe3506f4d41c53b008304d086a9e00d4b8 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 17 Jul 2025 17:05:19 +0200 Subject: [PATCH 18/32] python-binding unit test split, socket fixes Unit test portion split. Previously commented-out tests are now active. Fixes to the socket node resolved previous issues during testing. These arised when C-API functions were called through the bindings in likely unintended ways, causing undefined behavior (unexpected return values), segfaults or other memory related crashes. Fixes: - Socket descriptor (`sd`) is now initialized with -1 - `in` and `out` buffers are initialized with `nullptr` - Defensive deallocation and cleanup of buffers: * Check buffers before freeing * Zero memory before freeing * Set pointers to `nullptr` after freeing Signed-off-by: Kevin Vu te Laar --- lib/nodes/socket.cpp | 26 +++- tests/unit/python_binding.py | 289 ++++++++++++----------------------- 2 files changed, 121 insertions(+), 194 deletions(-) diff --git a/lib/nodes/socket.cpp b/lib/nodes/socket.cpp index a38a5afa7..284f302ce 100644 --- a/lib/nodes/socket.cpp +++ b/lib/nodes/socket.cpp @@ -66,6 +66,9 @@ int villas::node::socket_init(NodeCompat *n) { auto *s = n->getData(); s->formatter = nullptr; + s->sd = -1; + s->in.buf = nullptr; + s->out.buf = nullptr; return 0; } @@ -76,6 +79,17 @@ int villas::node::socket_destroy(NodeCompat *n) { if (s->formatter) delete s->formatter; + if (s->in.buf != nullptr) { + memset(s->in.buf, 0, s->in.buflen); + delete[] s->in.buf; + s->in.buf = nullptr; + } + if (s->out.buf != nullptr) { + memset(s->out.buf, 0, s->out.buflen); + delete[] s->out.buf; + s->out.buf = nullptr; + } + return 0; } @@ -354,8 +368,16 @@ int villas::node::socket_stop(NodeCompat *n) { return ret; } - delete[] s->in.buf; - delete[] s->out.buf; + if (s->in.buf != nullptr) { + memset(s->in.buf, 0, s->in.buflen); + delete[] s->in.buf; + s->in.buf = nullptr; + } + if (s->out.buf != nullptr) { + memset(s->out.buf, 0, s->out.buflen); + delete[] s->out.buf; + s->out.buf = nullptr; + } return 0; } diff --git a/tests/unit/python_binding.py b/tests/unit/python_binding.py index f528873cc..ed2e9fc9e 100644 --- a/tests/unit/python_binding.py +++ b/tests/unit/python_binding.py @@ -20,162 +20,147 @@ def setUp(self): except Exception as e: self.fail(f"new_node err: {e}") - def test_activity_changes(self): + def tearDown(self): + try: + vn.node_stop(self.test_node) + vn.node_destroy(self.test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_start(self): try: - vn.node_check(self.test_node) - vn.node_prepare(self.test_node) - # starting twice self.assertEqual(0, vn.node_start(self.test_node)) + # socket node will cause a RuntimeError + # the behavior is not consistent for each node # with self.assertRaises((AssertionError, RuntimeError)): # vn.node_start(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_new(self): + try: + node_uuid = str(uuid.uuid4()) + node_config = json.dumps(test_node_config, indent=2) + node = vn.node_new(node_uuid, node_config) + self.assertIsNotNone(node) + except Exception as e: + self.fail(f"err: {e}") + + def test_check(self): + try: + vn.node_check(self.test_node) + except Exception as e: + self.fail(f"err: {e}") - # check if the node is running + def test_prepare(self): + try: + vn.node_prepare(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_is_enabled(self): + try: self.assertTrue(vn.node_is_enabled(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") - # pausing twice - self.assertEqual(0, vn.node_pause(self.test_node)) + def test_pause(self): + try: + self.assertEqual(-1, vn.node_pause(self.test_node)) self.assertEqual(-1, vn.node_pause(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") - # resuming + def test_resume(self): + try: self.assertEqual(0, vn.node_resume(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") - # stopping twice + def test_stop(self): + try: self.assertEqual(0, vn.node_stop(self.test_node)) - # self.assertEqual(-1, vn.node_stop(self.test_node)) - - # # restarting - # vn.node_restart(self.test_node) - - # check if everything still works after restarting - # vn.node_pause(self.test_node) - # vn.node_resume(self.test_node) - # vn.node_stop(self.test_node) + self.assertEqual(0, vn.node_stop(self.test_node)) + vn.node_restart(self.test_node) + except Exception as e: + self.fail(f"err: {e}") - # terminating the node - vn.node_destroy(self.test_node) + def test_restart(self): + try: + self.assertEqual(0, vn.node_restart(self.test_node)) + self.assertEqual(0, vn.node_restart(self.test_node)) except Exception as e: - self.fail(f" err: {e}") + self.fail(f"err: {e}") - def test_details(self): + def test_node_name(self): try: # remove color codes before checking for equality self.assertEqual( "test_node(socket)", re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name(self.test_node)), ) + except Exception as e: + self.fail(f"err: {e}") - # node_name_short is bugged - # self.assertEqual('', vn.node_name_short(self.test_node)) + def test_node_name_short(self): + try: + print() + print(f"node name short: {vn.node_name_short(self.test_node)}") + print() + self.assertEqual("test_node", vn.node_name_short(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_full(self): + try: self.assertEqual( - vn.node_name(self.test_node) + "test_node(socket)" + ": uuid=" + self.node_uuid + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - vn.node_name_full(self.test_node), + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(self.test_node)), ) + except Exception as e: + self.fail(f"err: {e}") + def test_details(self): + try: self.assertEqual( "layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", vn.node_details(self.test_node), ) + except Exception as e: + self.fail(f"err: {e}") + def test_input_signals_max_cnt(self): + try: self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) except Exception as e: - self.fail(f" err: {e}") + self.fail(f"err: {e}") - # Test whether or not the json object is created by the wrapper and the module - def test_json_obj(self): + def test_node_output_signals_max_cnt(self): try: - node_config = vn.node_to_json(self.test_node) - node_config_str = json.dumps(node_config) - node_config_parsed = json.loads(node_config_str) - - self.assertEqual(node_config, node_config_parsed) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) except Exception as e: - self.fail(f" err: {e}") + self.fail(f"err: {e}") - # Test whether or not a node can be recreated with the string from node_to_json_str - # node_to_json_str has a wrong config format, thus the name config string will create, as of now, - # a node without a name - # uuid can not match - def test_config_from_string(self): + def test_node_is_valid_name(self): try: - config_str = vn.node_to_json_str(self.test_node) - config_obj = json.loads(config_str) - - config_copy_str = json.dumps(config_obj, indent=2) - - test_node = vn.node_new("", config_copy_str) - - self.assertEqual( - re.sub( - r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) - ), - re.sub( - r"^[^:]+: uuid=[0-9a-fA-F-]+, ", - "", - vn.node_name_full(self.test_node), - ), - ) + self.assertFalse(vn.node_is_valid_name("")) + self.assertFalse(vn.node_is_valid_name("###")) + self.assertFalse(vn.node_is_valid_name("v@l:d T3xt w;th invalid symb#ls")) + self.assertFalse(vn.node_is_valid_name("33_characters_long_string_invalid")) + self.assertTrue(vn.node_is_valid_name("32_characters_long_strings_valid")) + self.assertTrue(vn.node_is_valid_name("valid_name")) except Exception as e: - self.fail(f" err: {e}") + self.fail(f"err: {e}") - def test_rw_socket(self): + def test_reverse(self): try: - data_str = json.dumps(send_recv_test, indent=2) - data = json.loads(data_str) - - test_nodes = {} - for name, content in data.items(): - obj = {name: content} - config = json.dumps(obj, indent=2) - id = str(uuid.uuid4()) - - test_nodes[name] = vn.node_new(id, config) - - for node in test_nodes.values(): - if vn.node_check(node): - raise RuntimeError("Failed to verify node configuration") - if vn.node_prepare(node): - raise RuntimeError( - f"Failed to verify {vn.node_name(node)} node configuration" - ) - vn.node_start(node) - - # Arrays to store samples - send_smpls = vn.smps_array(1) - intmdt_smpls = vn.smps_array(100) - recv_smpls = vn.smps_array(100) - - for i in range(100): - send_smpls[0] = vn.sample_alloc(2) - intmdt_smpls[i] = vn.sample_alloc(2) - recv_smpls[i] = vn.sample_alloc(2) - - # Generate signals and send over send_socket - self.assertEqual( - vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1 - ) - self.assertEqual( - vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 - ) - - # read received signals and send them to recv_socket - self.assertEqual( - vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 - ) - self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 - ) - - # confirm rev_socket signals - self.assertEqual( - vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 - ) - + self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, vn.node_reverse(self.test_node)) except Exception as e: - self.fail(f" err: {e}") + self.fail(f"err: {e}") test_node_config = { @@ -191,85 +176,5 @@ def test_rw_socket(self): } } -send_recv_test = { - "send_socket": { - "type": "socket", - "format": "protobuf", - "layer": "udp", - "in": { - "address": "127.0.0.1:65532", - "signals": [ - {"name": "voltage", "type": "float", "unit": "V"}, - {"name": "current", "type": "float", "unit": "A"}, - ], - }, - "out": { - "address": "127.0.0.1:65533", - "netem": {"enabled": False}, - "multicast": {"enabled": False}, - }, - }, - "intmdt_socket": { - "type": "socket", - "format": "protobuf", - "layer": "udp", - "in": { - "address": "127.0.0.1:65533", - "signals": [ - {"name": "voltage", "type": "float", "unit": "V"}, - {"name": "current", "type": "float", "unit": "A"}, - ], - }, - "out": { - "address": "127.0.0.1:65534", - "netem": {"enabled": False}, - "multicast": {"enabled": False}, - }, - }, - "recv_socket": { - "type": "socket", - "format": "protobuf", - "layer": "udp", - "in": { - "address": "127.0.0.1:65534", - "signals": [ - {"name": "voltage", "type": "float", "unit": "V"}, - {"name": "current", "type": "float", "unit": "A"}, - ], - }, - "out": { - "address": "127.0.0.1:65535", - "netem": {"enabled": False}, - "multicast": {"enabled": False}, - }, - }, - "signal_generator": { - "type": "signal.v2", - "limit": 100, - "rate": 10, - "in": { - "signals": [ - { - "amplitude": 2, - "name": "voltage", - "phase": 90, - "signal": "sine", - "type": "float", - "unit": "V", - }, - { - "amplitude": 1, - "name": "current", - "phase": 0, - "signal": "sine", - "type": "float", - "unit": "A", - }, - ], - "hooks": [{"type": "print", "format": "villas.human"}], - }, - }, -} - if __name__ == "__main__": unittest.main() From 22714fe715e35fa301ed9ace2f5341ee8d2c3783 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 8 Sep 2025 11:10:45 +0200 Subject: [PATCH 19/32] unit tests: add test and enable test discovery Unit Tests in `tests/unit/python/` are now executed automatically as long as their filenames start with the prefix `test`. The binding unit test was renamed/moved from `tests/unit/python_binding.py` to `tests/unit/python/test_python_binding.py`. Node implementations return `-1` for functions that are not implemented. This is now covered for the `node_reverse()` function of the `socket` node. Signed-off-by: Kevin Vu te Laar --- tests/unit/CMakeLists.txt | 2 +- tests/unit/python/__init__.py | 0 .../test_python_binding.py} | 48 ++++++++++++++++--- 3 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 tests/unit/python/__init__.py rename tests/unit/{python_binding.py => python/test_python_binding.py} (79%) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 3a1394cb1..83cae7d38 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -37,7 +37,7 @@ add_custom_target(run-unit-tests add_custom_target(run-python-binding-tests COMMAND - python3 ${CMAKE_SOURCE_DIR}/tests/unit/python_binding.py + python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS python-binding USES_TERMINAL diff --git a/tests/unit/python/__init__.py b/tests/unit/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/python_binding.py b/tests/unit/python/test_python_binding.py similarity index 79% rename from tests/unit/python_binding.py rename to tests/unit/python/test_python_binding.py index ed2e9fc9e..7259c00bb 100644 --- a/tests/unit/python_binding.py +++ b/tests/unit/python/test_python_binding.py @@ -11,12 +11,15 @@ import villas_node as vn -class SimpleWrapperTests(unittest.TestCase): +class BindingUnitTests(unittest.TestCase): def setUp(self): try: self.node_uuid = str(uuid.uuid4()) self.config = json.dumps(test_node_config, indent=2) self.test_node = vn.node_new(self.node_uuid, self.config) + node_uuid = str(uuid.uuid4()) + config = json.dumps(signal_test_node_config, indent=2) + self.signal_test_node = vn.node_new(node_uuid, config) except Exception as e: self.fail(f"new_node err: {e}") @@ -30,10 +33,11 @@ def tearDown(self): def test_start(self): try: self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, vn.node_start(self.signal_test_node)) # socket node will cause a RuntimeError # the behavior is not consistent for each node - # with self.assertRaises((AssertionError, RuntimeError)): - # vn.node_start(self.test_node) + with self.assertRaises((AssertionError, RuntimeError)): + vn.node_start(self.test_node) except Exception as e: self.fail(f"err: {e}") @@ -81,7 +85,6 @@ def test_stop(self): try: self.assertEqual(0, vn.node_stop(self.test_node)) self.assertEqual(0, vn.node_stop(self.test_node)) - vn.node_restart(self.test_node) except Exception as e: self.fail(f"err: {e}") @@ -104,9 +107,6 @@ def test_node_name(self): def test_node_name_short(self): try: - print() - print(f"node name short: {vn.node_name_short(self.test_node)}") - print() self.assertEqual("test_node", vn.node_name_short(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -157,8 +157,13 @@ def test_node_is_valid_name(self): def test_reverse(self): try: + # socket has reverse() implemented, expected return 0 self.assertEqual(0, vn.node_reverse(self.test_node)) self.assertEqual(0, vn.node_reverse(self.test_node)) + + # signal.v2 has not reverse() implemented, expected return 1 + self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) + self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) except Exception as e: self.fail(f"err: {e}") @@ -176,5 +181,34 @@ def test_reverse(self): } } +signal_test_node_config = { + "signal_test_node": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + } +} + if __name__ == "__main__": unittest.main() From 052f4196a8ab6c1d1609e1102f2d687410910369 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 8 Sep 2025 11:22:22 +0200 Subject: [PATCH 20/32] integration tests: added and enable test discovery integration tests are now: - split from unit tests - improved to cover more cases - automatically executed upon discovery Integration tests must have the filename prefix `test` to be discovered. Signed-off-by: Kevin Vu te Laar --- tests/integration/CMakeLists.txt | 10 +- tests/integration/python/__init__.py | 0 .../integration/python/test_python_binding.py | 282 ++++++++++++++++++ 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 tests/integration/python/__init__.py create mode 100644 tests/integration/python/test_python_binding.py diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 777ea498d..8ae93db1c 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -18,4 +18,12 @@ add_custom_target(run-integration-tests villas-hook ) -add_dependencies(run-tests run-integration-tests) +add_custom_target(run-python-integration-tests + COMMAND + python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS + python-binding + USES_TERMINAL +) + +add_dependencies(run-tests run-integration-tests run-python-integration-tests) diff --git a/tests/integration/python/__init__.py b/tests/integration/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/python/test_python_binding.py b/tests/integration/python/test_python_binding.py new file mode 100644 index 000000000..c4bdf7d88 --- /dev/null +++ b/tests/integration/python/test_python_binding.py @@ -0,0 +1,282 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +import villas_node as vn + + +class BindingIntegrationTests(unittest.TestCase): + def setUp(self): + try: + self.node_uuid = str(uuid.uuid4()) + self.config = json.dumps(test_node_config, indent=2) + self.test_node = vn.node_new(self.node_uuid, self.config) + except Exception as e: + self.fail(f"new_node err: {e}") + + def tearDown(self): + try: + vn.node_stop(self.test_node) + vn.node_destroy(self.test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_activity_changes(self): + try: + vn.node_check(self.test_node) + vn.node_prepare(self.test_node) + # starting twice + self.assertEqual(0, vn.node_start(self.test_node)) + + # check if the node is running + self.assertTrue(vn.node_is_enabled(self.test_node)) + + # pausing twice + self.assertEqual(0, vn.node_pause(self.test_node)) + self.assertEqual(-1, vn.node_pause(self.test_node)) + + # resuming + self.assertEqual(0, vn.node_resume(self.test_node)) + + # stopping twice + self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, vn.node_stop(self.test_node)) + + # restarting + vn.node_restart(self.test_node) + + # check if everything still works after restarting + vn.node_pause(self.test_node) + vn.node_resume(self.test_node) + vn.node_stop(self.test_node) + vn.node_start(self.test_node) + except Exception as e: + self.fail(f" err: {e}") + + def test_reverse_node(self): + try: + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + + self.assertEqual(0, vn.node_reverse(self.test_node)) + + # input and output hooks/details are not reversed + # input and output are reversed, can be seen with wireshark and + # function test_rw_socket_and_reverse() below + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + # self.assertEqual(0, vn.node_input_signals_max_cnt(self.test_node)) + # self.assertEqual(1, vn.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"Reversing node in and output failed: {e}") + + # Test whether or not a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string to create + # a node without a name + # uuid can not match + def test_config_from_string(self): + try: + config_str = vn.node_to_json_str(self.test_node) + config_obj = json.loads(config_str) + + config_copy_str = json.dumps(config_obj, indent=2) + + test_node = vn.node_new("", config_copy_str) + + self.assertEqual( + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) + ), + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + vn.node_name_full(self.test_node), + ), + ) + except Exception as e: + self.fail(f" err: {e}") + + def test_rw_socket_and_reverse(self): + try: + data_str = json.dumps(send_recv_test, indent=2) + data = json.loads(data_str) + + test_nodes = {} + for name, content in data.items(): + obj = {name: content} + config = json.dumps(obj, indent=2) + id = str(uuid.uuid4()) + + test_nodes[name] = vn.node_new(id, config) + + for node in test_nodes.values(): + if vn.node_check(node): + raise RuntimeError("Failed to verify node configuration") + if vn.node_prepare(node): + raise RuntimeError( + f"Failed to verify {vn.node_name(node)} node configuration" + ) + vn.node_start(node) + + # Arrays to store samples + send_smpls = vn.smps_array(1) + intmdt_smpls = vn.smps_array(100) + recv_smpls = vn.smps_array(100) + + for i in range(100): + # send_smpls holds a new sample each time, but the old one still has a reference in the socket buffer (below) + # it is necessary to allocate a new sample each time to send_smpls + send_smpls[0] = vn.sample_alloc(2) + intmdt_smpls[i] = vn.sample_alloc(2) + recv_smpls[i] = vn.sample_alloc(2) + + # Generate signals and send over send_socket + self.assertEqual( + vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1 + ) + self.assertEqual( + vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + ) + + # read received signals and send them to recv_socket + self.assertEqual( + vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + ) + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + ) + + # confirm rev_socket signals + self.assertEqual( + vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) + + # reversing in and outputs + # stopping the socket is necessary to clean up buffers + # starting the node again will bind the reversed socket addresses + # this can be confirmed when observing network traffic + # node details do not represent this properly as of now + for node in test_nodes.values(): + vn.node_reverse(node) + vn.node_stop(node) + + for node in test_nodes.values(): + vn.node_start(node) + + self.assertEqual( + vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + ) + + # cleanup + for node in test_nodes.values(): + vn.node_stop(node) + vn.node_destroy(node) + + except Exception as e: + self.fail(f" err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +send_recv_test = { + "send_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65532", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65533", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "intmdt_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65533", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65534", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "recv_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65534", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65535", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "signal_generator": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + }, +} + +if __name__ == "__main__": + unittest.main() From be2b7701672ae5f802ab68bdaf5f8ae55206c836 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 9 Sep 2025 14:51:22 +0200 Subject: [PATCH 21/32] python binding: add wrapper and binding tweaks - Added a python-wrapper for the python-binding: * Automatic memory management * Restricted access and standardized sample handling * Added documentation via docstrings Binding Tweaks: * Support bulk allocation * Support writing sample slices/blocks Signed-off-by: Kevin Vu te Laar --- python/binding/villas-python-binding.cpp | 56 ++-- python/villas/node/binding.py | 373 +++++++++++++++++++++++ 2 files changed, 405 insertions(+), 24 deletions(-) create mode 100644 python/villas/node/binding.py diff --git a/python/binding/villas-python-binding.cpp b/python/binding/villas-python-binding.cpp index ff97d2ab9..82108ae84 100644 --- a/python/binding/villas-python-binding.cpp +++ b/python/binding/villas-python-binding.cpp @@ -35,6 +35,18 @@ class Array { delete[] smps; } + void *get_block(unsigned int start) { return (void *)&smps[start]; } + + void bulk_alloc(unsigned int start_idx, unsigned int stop_idx, + unsigned int smpl_len) { + for (unsigned int i = start_idx; i < stop_idx; ++i) { + if (smps[i]) { + sample_decref(smps[i]); + } + smps[i] = sample_alloc(smpl_len); + } + } + vsample *&operator[](unsigned int idx) { return smps[idx]; } vsample *&operator[](unsigned int idx) const { return smps[idx]; } @@ -99,7 +111,7 @@ PYBIND11_MODULE(villas_node, m) { m.def( "node_new", - [](const char *id_str, const char *json_str) -> vnode * { + [](const char *json_str, const char *id_str) -> vnode * { json_error_t err; uuid_t id; @@ -139,6 +151,10 @@ PYBIND11_MODULE(villas_node, m) { return node_read((vnode *)n, a.get_smps(), cnt); }); + m.def("node_read", [](void *n, void *smpls, unsigned cnt) -> int { + return node_read((vnode *)n, (vsample **)smpls, cnt); + }); + m.def("node_restart", [](void *n) -> int { return node_restart((vnode *)n); }); @@ -151,17 +167,6 @@ PYBIND11_MODULE(villas_node, m) { m.def("node_stop", [](void *n) -> int { return node_stop((vnode *)n); }); - m.def("node_to_json", [](void *n) -> py::str { - auto json = reinterpret_cast(n)->toJson(); - char *json_str = json_dumps(json, 0); - auto py_str = py::str(json_str); - - json_decref(json); - free(json_str); - - return py_str; - }); - m.def("node_to_json_str", [](void *n) -> py::str { auto json = reinterpret_cast(n)->toJson(); char *json_str = json_dumps(json, 0); @@ -177,6 +182,10 @@ PYBIND11_MODULE(villas_node, m) { return node_write((vnode *)n, a.get_smps(), cnt); }); + m.def("node_write", [](void *n, void *smpls, unsigned cnt) -> int { + return node_write((vnode *)n, (vsample **)smpls, cnt); + }); + m.def( "smps_array", [](unsigned int len) -> Array * { return new Array(len); }, py::return_value_policy::take_ownership); @@ -209,18 +218,17 @@ PYBIND11_MODULE(villas_node, m) { .def(py::init(), py::arg("len")) .def("__getitem__", [](Array &a, unsigned int idx) { - if (idx >= a.size()) { - throw py::index_error("Index out of bounds"); - } + assert(idx < a.size() && "Index out of bounds"); return a[idx]; }) - .def("__setitem__", [](Array &a, unsigned int idx, void *smp) { - if (idx >= a.size()) { - throw py::index_error("Index out of bounds"); - } - if (a[idx]) { - sample_decref(a[idx]); - } - a[idx] = (vsample *)smp; - }); + .def("__setitem__", + [](Array &a, unsigned int idx, void *smp) { + assert(idx < a.size() && "Index out of bounds"); + if (a[idx]) { + sample_decref(a[idx]); + } + a[idx] = (vsample *)smp; + }) + .def("get_block", &Array::get_block) + .def("bulk_alloc", &Array::bulk_alloc); } diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py new file mode 100644 index 000000000..4259c966b --- /dev/null +++ b/python/villas/node/binding.py @@ -0,0 +1,373 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +from typing import Optional, Union + +import functools +import json +import logging + +import villas_node as vn + +logger = logging.getLogger("villas.node") + + +class SamplesArray: + """ + Wrapper for a block of samples with automatic memory management. + + Supports: + - Reading block slices in combination with node_read(). + - Writing block slices in combination with node_write(). + - Automatic (de-)allocation of samples. + + Notes: + - Block slices are a slices with step size 1 + """ + + def _bulk_allocation(self, start_idx: int, end_idx: int, smpl_length: int): + """ + Allocates a block of samples. + + Args: + start_idx (int): Starting Index of a block. + end_idx (int): One past the last index to allocate. + smpl_length (int): length of sample per slot + + Notes: + - Block is determined by `start_idx`, `end_idx` + - A sample can hold multiple signals. smpl_length corresponds to the number of signals. + """ + return self._smps.bulk_alloc(start_idx, end_idx, smpl_length) + + def _get_block_handle(self, start_idx: Optional[int] = None): + """ + Get a handle to a block of samples. + + Args: + start_idx (Optional[int]): Starting index of the block. Defaults to 0. + + Returns: + vsample**: Pointer/handle to the underlying block masked as `void *`. + """ + if start_idx is None: + return self._smps.get_block(0) + else: + return self._smps.get_block(start_idx) + + def __init__(self, length: int): + """ + Initialize a SamplesArray. + + Notes: + Each sample slot can hold one node::Sample allocated via node_read(). + node::Sample can contain multiple signals, depending on smpl_length. + """ + self._smps = vn.smps_array(length) + self._len = length + + def __len__(self): + """Returns the length of the SamplesArray, which corresponds to the amount of samples it holds.""" + return self._len + + def __getitem__(self, idx: Union[int, slice]): + """Return a tuple containing self and an index or slice for node operations.""" + if isinstance(idx, slice): + return (self, idx) + elif isinstance(idx, int): + return (self, idx) + else: + logger.warning("Improper array index") + raise ValueError("Improper Index") + + def __copy__(self): + """Disallow shallow copying.""" + raise RuntimeError("Copying SamplesArray is not allowed") + + def __deepcopy__(self): + """Disallow deep copying.""" + raise RuntimeError("Copying SamplesArray is not allowed") + + +# helper functions + + +# function decorator for optional node_compat function calls +# that would return -1 if a function is not implemented +def _warn_if_not_implemented(func): + """ + Decorator to warn if specific node_* functions are not implemented and return -1. + + Returns: + Wrapping function that logs a warning if the return value is -1. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + ret = func(*args, **kwargs) + if ret == -1: + msg = f"[\033[33mWarning\033[0m]: Function '{func.__name__}()' is not implemented for node type '{vn.node_name(*args)}'." + logger.warning(msg) + return ret + + return wrapper + + +# node API bindings + + +def memory_init(*args): + return vn.memory_init(*args) + + +def node_check(node): + """Check node.""" + return vn.node_check(node) + + +def node_destroy(node): + """ + Delete a node. + + Notes: + - Note should be stopped first. + """ + return vn.node_destroy(node) + + +def node_details(node): + """Get node details.""" + return vn.node_details(node) + + +def node_input_signals_max_cnt(node): + """Get max input signal count.""" + return vn.node_input_signals_max_cnt(node) + + +def node_is_enabled(node): + """Check whether or not node is enabled.""" + return vn.node_is_enabled(node) + + +def node_is_valid_name(name: str): + """Check if a name can be used for a node.""" + return vn.node_is_valid_name(name) + + +def node_name(node): + """Get node name.""" + return vn.node_name(node) + + +def node_name_full(node): + """Get node name with full details.""" + return vn.node_name_full(node) + + +def node_name_short(node): + """Get node name with less details.""" + return vn.node_name_short(node) + + +def node_netem_fds(*args): + return vn.node_netem_fds(*args) + + +def node_new(config, uuid: Optional[str] = None): + """ + Create a new node. + + Args: + config (json/str): Configuration of the node. + uuid (Optional[str]): Unique identifier of the node. If `None`/empty, VILLASnode will assign one by default. + + Returns: + vnode *: Handle to a node. + """ + if uuid is None: + return vn.node_new(config, "") + else: + return vn.node_new(config, uuid) + + +def node_output_signals_max_cnt(node): + return vn.node_output_signals_max_cnt(node) + + +def node_pause(node): + """Pause a node""" + return vn.node_pause(node) + + +def node_poll_fds(*args): + return vn.node_poll_fds(*args) + + +def node_prepare(node): + """Prepare a node""" + return vn.node_prepare(node) + + +@_warn_if_not_implemented +def node_read(node, samples, sample_length, count): + """ + Read samples from a node into a SamplesArray or a block slice of its samples. + + Args: + node: Node handle. + samples: Either a SamplesArray or a tuple of (SamplesArray, index/slice). + sample_length: Length of each sample (number of signals). + count: Number of samples to read. + + Returns: + int: Number of samples read on success or -1 if not implemented. + + Notes: + - Return value may vary depending on node type. + - This function may be blocking. + """ + if isinstance(samples, SamplesArray): + samples._bulk_allocation(0, len(samples), sample_length) + return vn.node_read(node, samples._get_block_handle(0), count) + elif isinstance(samples, tuple): + smpls = samples[0] + if not isinstance(samples[1], slice): + raise ValueError("Invalid samples Parameter") + start, stop, _ = samples[1].indices(len(smpls)) + + # check for length mismatch + if (stop - start) != count: + raise ValueError("Sample slice length and sample count do not match.") + # check if out of bounds + if stop > len(smpls): + raise IndexError("Out of bounds") + + # allocate new samples and get block handle + samples[0]._bulk_allocation(start, stop, sample_length) + handle = samples[0]._get_block_handle(start) + + return vn.node_read(node, handle, count) + else: + logger.warning("Invalid samples Parameter") + return -1 + + +def node_restart(node): + """Restart a node.""" + return vn.node_restart(node) + + +def node_resume(node): + """Resume a node.""" + return vn.node_resume(node) + + +@_warn_if_not_implemented +def node_reverse(node): + """ + Reverse node input and output. + + Notes: + - Hooks are not reversed. + - Some nodes should be stopped or restarted before reversing. Especially nodes with in-/output buffers. + """ + return vn.node_reverse(node) + + +def node_start(node): + """ + Start a node. + + Notes: + - Nodes are not meant to be started again without stopping first. + """ + return vn.node_start(node) + + +def node_stop(node): + """ + Stop a node. + + Notes: + - Use before starting a node again. + """ + return vn.node_stop(node) + + +def node_to_json(node): + """ + Return the node configuration as json object. + + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + json_str = vn.node_to_json_str(node) + json_obj = json.loads(json_str) + return json_obj + + +def node_to_json_str(node): + """ + Returns the node configuration as string. + + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + return vn.node_to_json_str(node) + + +@_warn_if_not_implemented +def node_write(node, samples, count): + """ + Write samples from a SamplesArray, fully or as block slice, into a node. + + Args: + node: Node handle. + samples: Either a SamplesArray or a tuple of (SamplesArray, index/slice). + count: Number of samples to write. + + Returns: + int: Number of samples written on success, or -1 if not implemented. + + Notes: + - Return value may vary depending on node type. + """ + if isinstance(samples, SamplesArray): + return vn.node_write(node, samples._get_block_handle(), count) + elif isinstance(samples, tuple): + smpls = samples[0] + if not isinstance(samples[1], slice): + raise ValueError("Invalid samples Parameter") + start, stop, _ = samples[1].indices(len(smpls)) + + # check for length mismatch + if (stop - start) != count: + raise ValueError("Sample slice length and sample count do not match.") + # check for out of bounds + if stop > len(smpls): + raise IndexError("Out of bounds") + + # get block handle + handle = samples[0]._get_block_handle(start) + + return vn.node_write(node, handle, count) + else: + logger.warning("Invalid samples Parameter") + + +def sample_length(*args): + vn.sample_length(*args) + + +def sample_pack(*args): + vn.sample_pack(*args) + + +def sample_unpack(*args): + vn.sample_unpack(*args) From 5b887a4f12b2787eb649f9ed6949472af0dbaa8a Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 9 Sep 2025 15:38:18 +0200 Subject: [PATCH 22/32] binding tests: add binding wrapper tests - CI/CD integration of binding wrapper tests. - Add unit and integration tests for binding wrapper. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 2 +- .../python/test_binding_wrapper.py | 284 ++++++++++++++++++ .../integration/python/test_python_binding.py | 8 +- tests/unit/python/test_binding_wrapper.py | 239 +++++++++++++++ tests/unit/python/test_python_binding.py | 30 +- 5 files changed, 550 insertions(+), 13 deletions(-) create mode 100644 tests/integration/python/test_binding_wrapper.py create mode 100644 tests/unit/python/test_binding_wrapper.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d85f48102..48a9a9777 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,7 +173,7 @@ test:unit: image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - export PYTHONPATH=$PYTHONPATH:${PWD}/build/clients/python-binding + - export PYTHONPATH=$PYTHONPATH:${PWD}/build/clients/python-binding/:${PWD}/python/villas/node/ - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-binding-tests needs: - job: "build:source: [fedora]" diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py new file mode 100644 index 000000000..d50eeced8 --- /dev/null +++ b/tests/integration/python/test_binding_wrapper.py @@ -0,0 +1,284 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +import binding as vn + + +class BindingWrapperIntegrationTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = vn.node_new(self.config, self.node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def tearDown(self): + try: + vn.node_stop(self.test_node) + vn.node_destroy(self.test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_activity_changes(self): + try: + vn.node_check(self.test_node) + vn.node_prepare(self.test_node) + # starting twice + self.assertEqual(0, vn.node_start(self.test_node)) + + # check if the node is running + self.assertTrue(vn.node_is_enabled(self.test_node)) + + # pausing twice + self.assertEqual(0, vn.node_pause(self.test_node)) + self.assertEqual(-1, vn.node_pause(self.test_node)) + + # resuming + self.assertEqual(0, vn.node_resume(self.test_node)) + + # stopping twice + self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, vn.node_stop(self.test_node)) + + # restarting + vn.node_restart(self.test_node) + + # check if everything still works after restarting + vn.node_pause(self.test_node) + vn.node_resume(self.test_node) + vn.node_stop(self.test_node) + vn.node_start(self.test_node) + except Exception as e: + self.fail(f" err: {e}") + + def test_reverse_node(self): + try: + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + + self.assertEqual(0, vn.node_reverse(self.test_node)) + + # input and output hooks/details are not reversed + # input and output are reversed, can be seen with wireshark and + # function test_rw_socket_and_reverse() below + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + # self.assertEqual(0, vn.node_input_signals_max_cnt(self.test_node)) + # self.assertEqual(1, vn.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"Reversing node in and output failed: {e}") + + # Test whether or not a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string to create + # a node without a name + # uuid can not match + def test_config_from_string(self): + try: + config_str = vn.node_to_json_str(self.test_node) + config_obj = json.loads(config_str) + + config_copy_str = json.dumps(config_obj, indent=2) + + test_node = vn.node_new(config_copy_str) + + self.assertEqual( + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) + ), + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + vn.node_name_full(self.test_node), + ), + ) + except Exception as e: + self.fail(f" err: {e}") + + def test_rw_socket_and_reverse(self): + try: + data_str = json.dumps(send_recv_test, indent=2) + data = json.loads(data_str) + + test_nodes = {} + for name, content in data.items(): + obj = {name: content} + config = json.dumps(obj, indent=2) + id = str(uuid.uuid4()) + + test_nodes[name] = vn.node_new(config, id) + + for node in test_nodes.values(): + if vn.node_check(node): + raise RuntimeError("Failed to verify node configuration") + if vn.node_prepare(node): + raise RuntimeError( + f"Failed to verify {vn.node_name(node)} node configuration" + ) + vn.node_start(node) + + # Arrays to store samples + send_smpls = vn.SamplesArray(1) + intmdt_smpls = vn.SamplesArray(100) + recv_smpls = vn.SamplesArray(100) + + for i in range(100): + # Generate signals and send over send_socket + self.assertEqual( + vn.node_read(test_nodes["signal_generator"], send_smpls, 2, 1), 1 + ) + self.assertEqual( + vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + ) + + # read received signals and send them to recv_socket + self.assertEqual( + vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 2, 100), 100 + ) + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50), 50 + ) + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50), 50 + ) + + # confirm rev_socket signals + self.assertEqual( + vn.node_read(test_nodes["recv_socket"], recv_smpls[0:50], 2, 50), 50 + ) + self.assertEqual( + vn.node_read(test_nodes["recv_socket"], recv_smpls[50:100], 2, 50), 50 + ) + + # reversing in and outputs + # stopping the socket is necessary to clean up buffers + # starting the node again will bind the reversed socket addresses + # this can be confirmed when observing network traffic + # node details do not represent this properly as of now + for node in test_nodes.values(): + vn.node_reverse(node) + vn.node_stop(node) + + for node in test_nodes.values(): + vn.node_start(node) + + # if another 100 samples have not been allocated, sending 200 at once is impossible with recv_smpls + self.assertEqual( + vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) + # try writing as full slice + self.assertEqual( + vn.node_write(test_nodes["intmdt_socket"], recv_smpls[0:100], 100), 100 + ) + + # cleanup + for node in test_nodes.values(): + vn.node_stop(node) + vn.node_destroy(node) + + except Exception as e: + self.fail(f" err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +send_recv_test = { + "send_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65532", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65533", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "intmdt_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65533", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65534", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "recv_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65534", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65535", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "signal_generator": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + }, +} + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/python/test_python_binding.py b/tests/integration/python/test_python_binding.py index c4bdf7d88..d6a3c8682 100644 --- a/tests/integration/python/test_python_binding.py +++ b/tests/integration/python/test_python_binding.py @@ -14,9 +14,9 @@ class BindingIntegrationTests(unittest.TestCase): def setUp(self): try: - self.node_uuid = str(uuid.uuid4()) self.config = json.dumps(test_node_config, indent=2) - self.test_node = vn.node_new(self.node_uuid, self.config) + self.node_uuid = str(uuid.uuid4()) + self.test_node = vn.node_new(self.config, self.node_uuid) except Exception as e: self.fail(f"new_node err: {e}") @@ -87,7 +87,7 @@ def test_config_from_string(self): config_copy_str = json.dumps(config_obj, indent=2) - test_node = vn.node_new("", config_copy_str) + test_node = vn.node_new(config_copy_str, "") self.assertEqual( re.sub( @@ -113,7 +113,7 @@ def test_rw_socket_and_reverse(self): config = json.dumps(obj, indent=2) id = str(uuid.uuid4()) - test_nodes[name] = vn.node_new(id, config) + test_nodes[name] = vn.node_new(config, id) for node in test_nodes.values(): if vn.node_check(node): diff --git a/tests/unit/python/test_binding_wrapper.py b/tests/unit/python/test_binding_wrapper.py new file mode 100644 index 000000000..9c1af5b84 --- /dev/null +++ b/tests/unit/python/test_binding_wrapper.py @@ -0,0 +1,239 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +import binding as vn + + +class BindingWrapperUnitTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = vn.node_new(self.config, self.node_uuid) + config = json.dumps(signal_test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + self.signal_test_node = vn.node_new(config, node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def tearDown(self): + try: + vn.node_stop(self.test_node) + vn.node_destroy(self.test_node) + vn.node_stop(self.signal_test_node) + vn.node_destroy(self.signal_test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_start(self): + try: + self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, vn.node_start(self.signal_test_node)) + except Exception as e: + self.fail(f"err: {e}") + + @unittest.skip( + """Starting a socket twice will result in a RuntimeError. + Thise will leave the socket IP bound and may mess with other tests. + The behavior is Node specific.""" + ) + def test_start(self): + try: + self.assertEqual(0, vn.node_start(self.test_node)) + with self.assertRaises((AssertionError, RuntimeError)): + vn.node_start(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_new(self): + try: + node_config = json.dumps(test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + node = vn.node_new(node_config, node_uuid) + self.assertIsNotNone(node) + except Exception as e: + self.fail(f"err: {e}") + + def test_check(self): + try: + vn.node_check(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_prepare(self): + try: + vn.node_prepare(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_is_enabled(self): + try: + self.assertTrue(vn.node_is_enabled(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_pause(self): + try: + self.assertEqual(-1, vn.node_pause(self.test_node)) + self.assertEqual(-1, vn.node_pause(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_resume(self): + try: + self.assertEqual(0, vn.node_resume(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_stop(self): + try: + self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, vn.node_stop(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_restart(self): + try: + self.assertEqual(0, vn.node_restart(self.test_node)) + self.assertEqual(0, vn.node_restart(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name(self): + try: + # remove color codes before checking for equality + self.assertEqual( + "test_node(socket)", + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name(self.test_node)), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_short(self): + try: + self.assertEqual("test_node", vn.node_name_short(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_full(self): + try: + self.assertEqual( + "test_node(socket)" + + ": uuid=" + + self.node_uuid + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(self.test_node)), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_details(self): + try: + self.assertEqual( + "layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + vn.node_details(self.test_node), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_to_json(self): + try: + if not isinstance(vn.node_to_json(self.test_node), dict): + self.fail("Not a JSON object (dict)") + except Exception as e: + self.fail(f"err: {e}") + + def test_node_to_json_str(self): + try: + json.loads(vn.node_to_json_str(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_input_signals_max_cnt(self): + try: + self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_output_signals_max_cnt(self): + try: + self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_is_valid_name(self): + try: + self.assertFalse(vn.node_is_valid_name("")) + self.assertFalse(vn.node_is_valid_name("###")) + self.assertFalse(vn.node_is_valid_name("v@l:d T3xt w;th invalid symb#ls")) + self.assertFalse(vn.node_is_valid_name("33_characters_long_string_invalid")) + self.assertTrue(vn.node_is_valid_name("32_characters_long_strings_valid")) + self.assertTrue(vn.node_is_valid_name("valid_name")) + except Exception as e: + self.fail(f"err: {e}") + + def test_reverse(self): + try: + # socket has reverse() implemented, expected return 0 + self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, vn.node_reverse(self.test_node)) + + # signal.v2 has not reverse() implemented, expected return 1 + self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) + self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) + except Exception as e: + self.fail(f"err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +signal_test_node_config = { + "signal_test_node": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + } +} + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/test_python_binding.py b/tests/unit/python/test_python_binding.py index 7259c00bb..88844942d 100644 --- a/tests/unit/python/test_python_binding.py +++ b/tests/unit/python/test_python_binding.py @@ -14,12 +14,12 @@ class BindingUnitTests(unittest.TestCase): def setUp(self): try: - self.node_uuid = str(uuid.uuid4()) self.config = json.dumps(test_node_config, indent=2) - self.test_node = vn.node_new(self.node_uuid, self.config) - node_uuid = str(uuid.uuid4()) + self.node_uuid = str(uuid.uuid4()) + self.test_node = vn.node_new(self.config, self.node_uuid) config = json.dumps(signal_test_node_config, indent=2) - self.signal_test_node = vn.node_new(node_uuid, config) + node_uuid = str(uuid.uuid4()) + self.signal_test_node = vn.node_new(config, node_uuid) except Exception as e: self.fail(f"new_node err: {e}") @@ -27,6 +27,8 @@ def tearDown(self): try: vn.node_stop(self.test_node) vn.node_destroy(self.test_node) + vn.node_stop(self.signal_test_node) + vn.node_destroy(self.signal_test_node) except Exception as e: self.fail(f"node cleanup error: {e}") @@ -34,8 +36,19 @@ def test_start(self): try: self.assertEqual(0, vn.node_start(self.test_node)) self.assertEqual(0, vn.node_start(self.signal_test_node)) - # socket node will cause a RuntimeError - # the behavior is not consistent for each node + except Exception as e: + self.fail(f"err: {e}") + + @unittest.skip( + """ + Starting a socket twice will result in a RuntimeError. + Thise will leave the socket IP bound and may mess with other tests. + The behavior is Node specific. + """ + ) + def test_start(self): + try: + self.assertEqual(0, vn.node_start(self.test_node)) with self.assertRaises((AssertionError, RuntimeError)): vn.node_start(self.test_node) except Exception as e: @@ -43,9 +56,9 @@ def test_start(self): def test_new(self): try: - node_uuid = str(uuid.uuid4()) node_config = json.dumps(test_node_config, indent=2) - node = vn.node_new(node_uuid, node_config) + node_uuid = str(uuid.uuid4()) + node = vn.node_new(node_config, node_uuid) self.assertIsNotNone(node) except Exception as e: self.fail(f"err: {e}") @@ -83,6 +96,7 @@ def test_resume(self): def test_stop(self): try: + self.assertEqual(0, vn.node_start(self.test_node)) self.assertEqual(0, vn.node_stop(self.test_node)) self.assertEqual(0, vn.node_stop(self.test_node)) except Exception as e: From 279eb80c4ee1f388f782429c6180fb1e8ba1f4e1 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 9 Sep 2025 18:05:36 +0200 Subject: [PATCH 23/32] binding tests: CI fix and reformat - Updated PYTHONPATH to match directory structure. - Python integration tests are now executed. - Binding related `.py`-files now adhere to 79-character line length. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 7 ++- python/villas/node/binding.py | 44 ++++++++------ .../python/test_binding_wrapper.py | 58 +++++++++++++------ .../integration/python/test_python_binding.py | 37 +++++++----- tests/unit/CMakeLists.txt | 4 +- tests/unit/python/test_binding_wrapper.py | 37 ++++++++---- tests/unit/python/test_python_binding.py | 35 +++++++---- 7 files changed, 146 insertions(+), 76 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 48a9a9777..cdf842591 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -173,8 +173,8 @@ test:unit: image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - export PYTHONPATH=$PYTHONPATH:${PWD}/build/clients/python-binding/:${PWD}/python/villas/node/ - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-binding-tests + - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/ + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-unit-tests needs: - job: "build:source: [fedora]" artifacts: true @@ -184,7 +184,8 @@ test:integration: image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests + - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/ + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests run-python-integration-tests artifacts: name: ${CI_PROJECT_NAME}-integration-tests-${CI_BUILD_REF} when: always diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py index 4259c966b..cce550a08 100644 --- a/python/villas/node/binding.py +++ b/python/villas/node/binding.py @@ -20,8 +20,8 @@ class SamplesArray: Wrapper for a block of samples with automatic memory management. Supports: - - Reading block slices in combination with node_read(). - - Writing block slices in combination with node_write(). + - Reading block slices in combination with `node_read()`. + - Writing block slices in combination with `node_write()`. - Automatic (de-)allocation of samples. Notes: @@ -39,7 +39,8 @@ def _bulk_allocation(self, start_idx: int, end_idx: int, smpl_length: int): Notes: - Block is determined by `start_idx`, `end_idx` - - A sample can hold multiple signals. smpl_length corresponds to the number of signals. + - Samples can hold multiple signals. + - `smpl_length` corresponds to the number of signals. """ return self._smps.bulk_alloc(start_idx, end_idx, smpl_length) @@ -48,10 +49,10 @@ def _get_block_handle(self, start_idx: Optional[int] = None): Get a handle to a block of samples. Args: - start_idx (Optional[int]): Starting index of the block. Defaults to 0. + start_idx (Optional[int]): Starting index of the block. Default 0. Returns: - vsample**: Pointer/handle to the underlying block masked as `void *`. + vsample**: Handle to the underlying block masked as `void *`. """ if start_idx is None: return self._smps.get_block(0) @@ -63,18 +64,21 @@ def __init__(self, length: int): Initialize a SamplesArray. Notes: - Each sample slot can hold one node::Sample allocated via node_read(). - node::Sample can contain multiple signals, depending on smpl_length. + - Each sample slot can hold one sample allocated via `node_read()`. + - Sample can contain multiple signals, depending on `smpl_length`. """ self._smps = vn.smps_array(length) self._len = length def __len__(self): - """Returns the length of the SamplesArray, which corresponds to the amount of samples it holds.""" + """ + Returns the length of the SamplesArray. + Corresponds to the amount of samples it holds. + """ return self._len def __getitem__(self, idx: Union[int, slice]): - """Return a tuple containing self and an index or slice for node operations.""" + """Return tuple containing self and index/slice for node operations.""" if isinstance(idx, slice): return (self, idx) elif isinstance(idx, int): @@ -99,7 +103,7 @@ def __deepcopy__(self): # that would return -1 if a function is not implemented def _warn_if_not_implemented(func): """ - Decorator to warn if specific node_* functions are not implemented and return -1. + Decorator to warn if specific `node_*()` functions are not implemented. Returns: Wrapping function that logs a warning if the return value is -1. @@ -109,7 +113,8 @@ def _warn_if_not_implemented(func): def wrapper(*args, **kwargs): ret = func(*args, **kwargs) if ret == -1: - msg = f"[\033[33mWarning\033[0m]: Function '{func.__name__}()' is not implemented for node type '{vn.node_name(*args)}'." + msg = f"[\033[33mWarning\033[0m]: Function '{func.__name__}()' \ + is not implemented for node type '{vn.node_name(*args)}'." logger.warning(msg) return ret @@ -183,7 +188,8 @@ def node_new(config, uuid: Optional[str] = None): Args: config (json/str): Configuration of the node. - uuid (Optional[str]): Unique identifier of the node. If `None`/empty, VILLASnode will assign one by default. + uuid (Optional[str]): Unique identifier of the node. + If `None`, VILLASnode will assign one by default. Returns: vnode *: Handle to a node. @@ -215,11 +221,11 @@ def node_prepare(node): @_warn_if_not_implemented def node_read(node, samples, sample_length, count): """ - Read samples from a node into a SamplesArray or a block slice of its samples. + Read samples from a node into SamplesArray or a block slice of its samples. Args: node: Node handle. - samples: Either a SamplesArray or a tuple of (SamplesArray, index/slice). + samples: Either a SamplesArray or a tuple (SamplesArray, index/slice). sample_length: Length of each sample (number of signals). count: Number of samples to read. @@ -241,7 +247,7 @@ def node_read(node, samples, sample_length, count): # check for length mismatch if (stop - start) != count: - raise ValueError("Sample slice length and sample count do not match.") + raise ValueError("Slice length and sample count do not match.") # check if out of bounds if stop > len(smpls): raise IndexError("Out of bounds") @@ -273,7 +279,8 @@ def node_reverse(node): Notes: - Hooks are not reversed. - - Some nodes should be stopped or restarted before reversing. Especially nodes with in-/output buffers. + - Some nodes should be stopped or restarted before reversing. + - Nodes with in-/output buffers should be stopped before reversing. """ return vn.node_reverse(node) @@ -294,6 +301,7 @@ def node_stop(node): Notes: - Use before starting a node again. + - May delete in-/output buffers of a node. """ return vn.node_stop(node) @@ -329,7 +337,7 @@ def node_write(node, samples, count): Args: node: Node handle. - samples: Either a SamplesArray or a tuple of (SamplesArray, index/slice). + samples: Either a SamplesArray or a tuple (SamplesArray, index/slice). count: Number of samples to write. Returns: @@ -348,7 +356,7 @@ def node_write(node, samples, count): # check for length mismatch if (stop - start) != count: - raise ValueError("Sample slice length and sample count do not match.") + raise ValueError("Slice length and sample count do not match.") # check for out of bounds if stop > len(smpls): raise IndexError("Out of bounds") diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py index d50eeced8..8f278651b 100644 --- a/tests/integration/python/test_binding_wrapper.py +++ b/tests/integration/python/test_binding_wrapper.py @@ -71,14 +71,12 @@ def test_reverse_node(self): # function test_rw_socket_and_reverse() below self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) - # self.assertEqual(0, vn.node_input_signals_max_cnt(self.test_node)) - # self.assertEqual(1, vn.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"Reversing node in and output failed: {e}") - # Test whether or not a node can be recreated with the string from node_to_json_str - # node_to_json_str has a wrong config format causing the config string to create - # a node without a name + # Test if a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string + # to create a node without a name # uuid can not match def test_config_from_string(self): try: @@ -91,7 +89,9 @@ def test_config_from_string(self): self.assertEqual( re.sub( - r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + vn.node_name_full(test_node), ), re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", @@ -120,7 +120,7 @@ def test_rw_socket_and_reverse(self): raise RuntimeError("Failed to verify node configuration") if vn.node_prepare(node): raise RuntimeError( - f"Failed to verify {vn.node_name(node)} node configuration" + f"Failed to verify {vn.node_name(node)} node config" ) vn.node_start(node) @@ -132,7 +132,10 @@ def test_rw_socket_and_reverse(self): for i in range(100): # Generate signals and send over send_socket self.assertEqual( - vn.node_read(test_nodes["signal_generator"], send_smpls, 2, 1), 1 + vn.node_read( + test_nodes["signal_generator"], send_smpls, 2, 1 + ), + 1, ) self.assertEqual( vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 @@ -140,21 +143,36 @@ def test_rw_socket_and_reverse(self): # read received signals and send them to recv_socket self.assertEqual( - vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 2, 100), 100 + vn.node_read( + test_nodes["intmdt_socket"], intmdt_smpls, 2, 100 + ), + 100, ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50), 50 + vn.node_write( + test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50 + ), + 50, ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50), 50 + vn.node_write( + test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50 + ), + 50, ) # confirm rev_socket signals self.assertEqual( - vn.node_read(test_nodes["recv_socket"], recv_smpls[0:50], 2, 50), 50 + vn.node_read( + test_nodes["recv_socket"], recv_smpls[0:50], 2, 50 + ), + 50, ) self.assertEqual( - vn.node_read(test_nodes["recv_socket"], recv_smpls[50:100], 2, 50), 50 + vn.node_read( + test_nodes["recv_socket"], recv_smpls[50:100], 2, 50 + ), + 50, ) # reversing in and outputs @@ -169,13 +187,17 @@ def test_rw_socket_and_reverse(self): for node in test_nodes.values(): vn.node_start(node) - # if another 100 samples have not been allocated, sending 200 at once is impossible with recv_smpls + # if another 50 samples have not been allocated, + # sending 100 at once is impossible with recv_smpls self.assertEqual( vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 ) # try writing as full slice self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], recv_smpls[0:100], 100), 100 + vn.node_write( + test_nodes["intmdt_socket"], recv_smpls[0:100], 100 + ), + 100, ) # cleanup @@ -194,7 +216,9 @@ def test_rw_socket_and_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + "signals": [ + {"name": "tap_position", "type": "integer", "init": 0} + ], }, "out": {"address": "127.0.0.1:12001"}, } @@ -252,7 +276,7 @@ def test_rw_socket_and_reverse(self): "multicast": {"enabled": False}, }, }, - "signal_generator": { + "signal_gen": { "type": "signal.v2", "limit": 100, "rate": 10, diff --git a/tests/integration/python/test_python_binding.py b/tests/integration/python/test_python_binding.py index d6a3c8682..662f09d3f 100644 --- a/tests/integration/python/test_python_binding.py +++ b/tests/integration/python/test_python_binding.py @@ -71,14 +71,12 @@ def test_reverse_node(self): # function test_rw_socket_and_reverse() below self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) - # self.assertEqual(0, vn.node_input_signals_max_cnt(self.test_node)) - # self.assertEqual(1, vn.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"Reversing node in and output failed: {e}") - # Test whether or not a node can be recreated with the string from node_to_json_str - # node_to_json_str has a wrong config format causing the config string to create - # a node without a name + # Test if a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string + # to create a node without a name # uuid can not match def test_config_from_string(self): try: @@ -91,7 +89,9 @@ def test_config_from_string(self): self.assertEqual( re.sub( - r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", vn.node_name_full(test_node) + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + vn.node_name_full(test_node), ), re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", @@ -120,7 +120,7 @@ def test_rw_socket_and_reverse(self): raise RuntimeError("Failed to verify node configuration") if vn.node_prepare(node): raise RuntimeError( - f"Failed to verify {vn.node_name(node)} node configuration" + f"Failed to verify {vn.node_name(node)} node config" ) vn.node_start(node) @@ -130,15 +130,19 @@ def test_rw_socket_and_reverse(self): recv_smpls = vn.smps_array(100) for i in range(100): - # send_smpls holds a new sample each time, but the old one still has a reference in the socket buffer (below) - # it is necessary to allocate a new sample each time to send_smpls + # send_smpls holds a new sample each time, but the + # old one still has a reference in the socket buffer (below) + # it is necessary to allocate a new sample each time send_smpls[0] = vn.sample_alloc(2) intmdt_smpls[i] = vn.sample_alloc(2) recv_smpls[i] = vn.sample_alloc(2) # Generate signals and send over send_socket self.assertEqual( - vn.node_read(test_nodes["signal_generator"], send_smpls, 1), 1 + vn.node_read( + test_nodes["signal_generator"], send_smpls, 1 + ), + 1, ) self.assertEqual( vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 @@ -146,10 +150,12 @@ def test_rw_socket_and_reverse(self): # read received signals and send them to recv_socket self.assertEqual( - vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, ) # confirm rev_socket signals @@ -173,7 +179,8 @@ def test_rw_socket_and_reverse(self): vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100 + vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, ) # cleanup @@ -192,7 +199,9 @@ def test_rw_socket_and_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + "signals": [ + {"name": "tap_position", "type": "integer", "init": 0} + ], }, "out": {"address": "127.0.0.1:12001"}, } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 83cae7d38..d42b7e6a9 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,7 +35,7 @@ add_custom_target(run-unit-tests USES_TERMINAL ) -add_custom_target(run-python-binding-tests +add_custom_target(run-python-unit-tests COMMAND python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS @@ -44,4 +44,4 @@ add_custom_target(run-python-binding-tests ) add_dependencies(tests unit-tests) -add_dependencies(run-tests run-unit-tests run-python-binding-tests) +add_dependencies(run-tests run-unit-tests run-python-unit-tests) diff --git a/tests/unit/python/test_binding_wrapper.py b/tests/unit/python/test_binding_wrapper.py index 9c1af5b84..82827e38f 100644 --- a/tests/unit/python/test_binding_wrapper.py +++ b/tests/unit/python/test_binding_wrapper.py @@ -44,7 +44,7 @@ def test_start(self): Thise will leave the socket IP bound and may mess with other tests. The behavior is Node specific.""" ) - def test_start(self): + def test_start_err(self): try: self.assertEqual(0, vn.node_start(self.test_node)) with self.assertRaises((AssertionError, RuntimeError)): @@ -125,12 +125,15 @@ def test_node_name_short(self): def test_node_name_full(self): try: + node = self.test_node self.assertEqual( "test_node(socket)" + ": uuid=" + self.node_uuid - + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(self.test_node)), + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(node)), ) except Exception as e: self.fail(f"err: {e}") @@ -138,7 +141,9 @@ def test_node_name_full(self): def test_details(self): try: self.assertEqual( - "layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + "layer=udp, " + + "in.address=0.0.0.0:12000, " + + "out.address=127.0.0.1:12001", vn.node_details(self.test_node), ) except Exception as e: @@ -153,6 +158,8 @@ def test_node_to_json(self): def test_node_to_json_str(self): try: + print(vn.node_to_json_str(self.test_node)) + print(vn.node_to_json_str(self.test_node)) json.loads(vn.node_to_json_str(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -171,12 +178,18 @@ def test_node_output_signals_max_cnt(self): def test_node_is_valid_name(self): try: - self.assertFalse(vn.node_is_valid_name("")) - self.assertFalse(vn.node_is_valid_name("###")) - self.assertFalse(vn.node_is_valid_name("v@l:d T3xt w;th invalid symb#ls")) - self.assertFalse(vn.node_is_valid_name("33_characters_long_string_invalid")) - self.assertTrue(vn.node_is_valid_name("32_characters_long_strings_valid")) - self.assertTrue(vn.node_is_valid_name("valid_name")) + invalid_names = [ + "", + "###", + "v@l:d T3xt w;th invalid symb#ls", + "33_characters_long_string_invalid", + ] + valid_names = ["32_characters_long_strings_valid", "valid_name"] + + for name in invalid_names: + self.assertFalse(vn.node_is_valid_name(name)) + for name in valid_names: + self.assertTrue(vn.node_is_valid_name(name)) except Exception as e: self.fail(f"err: {e}") @@ -200,7 +213,9 @@ def test_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + "signals": [ + {"name": "tap_position", "type": "integer", "init": 0} + ], }, "out": {"address": "127.0.0.1:12001"}, } diff --git a/tests/unit/python/test_python_binding.py b/tests/unit/python/test_python_binding.py index 88844942d..01ac98c6c 100644 --- a/tests/unit/python/test_python_binding.py +++ b/tests/unit/python/test_python_binding.py @@ -46,7 +46,7 @@ def test_start(self): The behavior is Node specific. """ ) - def test_start(self): + def test_start_err(self): try: self.assertEqual(0, vn.node_start(self.test_node)) with self.assertRaises((AssertionError, RuntimeError)): @@ -127,12 +127,15 @@ def test_node_name_short(self): def test_node_name_full(self): try: + node = self.test_node self.assertEqual( "test_node(socket)" + ": uuid=" + self.node_uuid - + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0, in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(self.test_node)), + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(node)), ) except Exception as e: self.fail(f"err: {e}") @@ -140,7 +143,9 @@ def test_node_name_full(self): def test_details(self): try: self.assertEqual( - "layer=udp, in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + "layer=udp, " + + "in.address=0.0.0.0:12000, " + + "out.address=127.0.0.1:12001", vn.node_details(self.test_node), ) except Exception as e: @@ -160,12 +165,18 @@ def test_node_output_signals_max_cnt(self): def test_node_is_valid_name(self): try: - self.assertFalse(vn.node_is_valid_name("")) - self.assertFalse(vn.node_is_valid_name("###")) - self.assertFalse(vn.node_is_valid_name("v@l:d T3xt w;th invalid symb#ls")) - self.assertFalse(vn.node_is_valid_name("33_characters_long_string_invalid")) - self.assertTrue(vn.node_is_valid_name("32_characters_long_strings_valid")) - self.assertTrue(vn.node_is_valid_name("valid_name")) + invalid_names = [ + "", + "###", + "v@l:d T3xt w;th invalid symb#ls", + "33_characters_long_string_invalid", + ] + valid_names = ["32_characters_long_strings_valid", "valid_name"] + + for name in invalid_names: + self.assertFalse(vn.node_is_valid_name(name)) + for name in valid_names: + self.assertFalse(vn.node_is_valid_name(name)) except Exception as e: self.fail(f"err: {e}") @@ -189,7 +200,9 @@ def test_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + "signals": [ + {"name": "tap_position", "type": "integer", "init": 0} + ], }, "out": {"address": "127.0.0.1:12001"}, } From 66e1f14ffb19f50d53ce3d8002fda6431fdf6701 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Wed, 10 Sep 2025 05:33:01 +0200 Subject: [PATCH 24/32] binding wrapper: CI/build fix, add binding stubs CI: - Python unit and integration tests have their own CI test now. - Unit and integration test pipelines reverted to pre-binding related changes. Binding need workarounds to comply with packaging, CI and building compatibility. - Format fixes. - `binding.py` (binding wrapper) and bindings have stubs now. CMake build: - Naming is more straight forward. - Bindings install into `pythonX.Y/site-packages/villas/node/`. Should be compatible with a potential pypi-package related to the wrapper binding. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 28 ++++- python/binding/CMakeLists.txt | 22 ++-- ...on-binding.cpp => capi_python_binding.cpp} | 2 +- python/villas/node/binding.py | 2 +- python/villas/node/binding.pyi | 48 +++++++ python/villas/node/python_binding.pyi | 63 ++++++++++ tests/integration/CMakeLists.txt | 2 +- .../python/test_binding_wrapper.py | 118 ++++++++---------- .../integration/python/test_python_binding.py | 106 ++++++++-------- tests/unit/CMakeLists.txt | 2 +- tests/unit/python/test_binding_wrapper.py | 80 ++++++------ tests/unit/python/test_python_binding.py | 74 ++++++----- 12 files changed, 328 insertions(+), 219 deletions(-) rename python/binding/{villas-python-binding.cpp => capi_python_binding.cpp} (99%) create mode 100644 python/villas/node/binding.pyi create mode 100644 python/villas/node/python_binding.pyi diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdf842591..9b0682fee 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,13 +168,34 @@ test:cppcheck: - cppcheck.log expose_as: cppcheck +test:python_unit_integration: + stage: test + image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} + before_script: + # dependency from node.py, which gets imported because of __init__.py in node/villas/python/node + - pip install requests + script: + # build binding and symlink to villas.node folder + # helps with correct imports and compatiblity for CI, binding wrapper and binding install + - export PYTHONPATH=$PYTHONPATH:${PWD}/python + - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding + + - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1) + - link_path=${PWD}/python/villas/node/$(basename $python_binding) + - ln -sf $binding_path $link_path + + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests + - rm $node_path + needs: + - job: "build:source: [fedora]" + artifacts: true + test:unit: stage: test image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/ - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common run-python-unit-tests + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-unit-tests run-unit-tests-common needs: - job: "build:source: [fedora]" artifacts: true @@ -184,8 +205,7 @@ test:integration: image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} script: - cmake -S . -B build ${CMAKE_OPTS} - - export PYTHONPATH=$PYTHONPATH:${PWD}/build/python/binding/:${PWD}/python/villas/node/ - - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests run-python-integration-tests + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-integration-tests artifacts: name: ${CI_PROJECT_NAME}-integration-tests-${CI_BUILD_REF} when: always diff --git a/python/binding/CMakeLists.txt b/python/binding/CMakeLists.txt index fd645f338..bb81221ec 100644 --- a/python/binding/CMakeLists.txt +++ b/python/binding/CMakeLists.txt @@ -11,25 +11,29 @@ if(pybind11_FOUND) find_package(Python3 REQUIRED COMPONENTS Interpreter Development) execute_process( - COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('stdlib') + '/lib-dynload')" - OUTPUT_VARIABLE PYTHON_LIB_DYNLOAD_DIR + COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('purelib'))" + OUTPUT_VARIABLE PYTHON_SITE_PACKAGES OUTPUT_STRIP_TRAILING_WHITESPACE ) message(STATUS "Found Python version: ${Python_VERSION}") message(STATUS "Python major version: ${Python_VERSION_MAJOR}") message(STATUS "Python minor version: ${Python_VERSION_MINOR}") - message(STATUS "Python .so install directory: ${PYTHON_LIB_DYNLOAD_DIR}") - pybind11_add_module(python-binding villas-python-binding.cpp) - set_target_properties(python-binding PROPERTIES OUTPUT_NAME villas_node) - target_link_libraries(python-binding PUBLIC villas) + pybind11_add_module(python_binding capi_python_binding.cpp) + target_link_libraries(python_binding PUBLIC villas) install( - TARGETS python-binding + TARGETS python_binding COMPONENT lib - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${PYTHON_LIB_DYNLOAD_DIR} + LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/ + ) + file(WRITE "${CMAKE_BINARY_DIR}/python/villas/__init__.py" "") + file(WRITE "${CMAKE_BINARY_DIR}/python/villas/node/__init__.py" "") + install( + DIRECTORY "${CMAKE_BINARY_DIR}/python/villas" + DESTINATION "${PYTHON_SITE_PACKAGES}" + FILES_MATCHING PATTERN "__init__.py" ) else() message(STATUS "pybind11 not found. Skipping Python wrapper build.") diff --git a/python/binding/villas-python-binding.cpp b/python/binding/capi_python_binding.cpp similarity index 99% rename from python/binding/villas-python-binding.cpp rename to python/binding/capi_python_binding.cpp index 82108ae84..f5271b237 100644 --- a/python/binding/villas-python-binding.cpp +++ b/python/binding/capi_python_binding.cpp @@ -67,7 +67,7 @@ class Array { * @param villas_node Name of the module to be bound * @param m Access variable for modifying the module code */ -PYBIND11_MODULE(villas_node, m) { +PYBIND11_MODULE(python_binding, m) { m.def("memory_init", &memory_init); m.def("node_check", [](void *n) -> int { return node_check((vnode *)n); }); diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py index cce550a08..7556426d0 100644 --- a/python/villas/node/binding.py +++ b/python/villas/node/binding.py @@ -10,7 +10,7 @@ import json import logging -import villas_node as vn +import villas.node.python_binding as vn logger = logging.getLogger("villas.node") diff --git a/python/villas/node/binding.pyi b/python/villas/node/binding.pyi new file mode 100644 index 000000000..335be5e0d --- /dev/null +++ b/python/villas/node/binding.pyi @@ -0,0 +1,48 @@ +from _typeshed import Incomplete +from typing import Any, Callable + +logger: Incomplete + + +class SamplesArray: + def __init__(self, length: int) -> None: ... + def __len__(self) -> int: ... + def __getitem__(self, idx: int | slice): ... + def __copy__(self) -> None: ... + def __deepcopy__(self) -> None: ... + + +def _warn_if_not_implemented( + func: Callable[..., Any], +) -> Callable[..., Any]: ... +def memory_init(*args): ... +def node_check(node): ... +def node_destroy(node): ... +def node_details(node): ... +def node_input_signals_max_cnt(node): ... +def node_is_enabled(node): ... +def node_is_valid_name(name: str): ... +def node_name(node): ... +def node_name_full(node): ... +def node_name_short(node): ... +def node_netem_fds(*args): ... +def node_new(config, uuid: str | None = None): ... +def node_output_signals_max_cnt(node): ... +def node_pause(node): ... +def node_poll_fds(*args): ... +def node_prepare(node): ... +@_warn_if_not_implemented +def node_read(node, samples, sample_length, count): ... +def node_restart(node): ... +def node_resume(node): ... +@_warn_if_not_implemented +def node_reverse(node): ... +def node_start(node): ... +def node_stop(node): ... +def node_to_json(node): ... +def node_to_json_str(node): ... +@_warn_if_not_implemented +def node_write(node, samples, count): ... +def sample_length(*args) -> None: ... +def sample_pack(*args) -> None: ... +def sample_unpack(*args) -> None: ... diff --git a/python/villas/node/python_binding.pyi b/python/villas/node/python_binding.pyi new file mode 100644 index 000000000..309c61a6c --- /dev/null +++ b/python/villas/node/python_binding.pyi @@ -0,0 +1,63 @@ +import typing +from typing import Any, Callable, overload + +Array = Any +capsule = Any +timespec = Any + + +class SamplesArray: + def __init__(self, len: int) -> None: ... + def bulk_alloc(self, arg0: int, arg1: int, arg2: int) -> None: ... + def get_block(self, arg0: int) -> capsule: ... + def __getitem__(self, arg0: int) -> capsule: ... + def __iter__(self) -> typing.Iterator[capsule]: ... + def __setitem__(self, arg0: int, arg1: capsule) -> None: ... + + +def memory_init(arg0: int) -> int: ... +def node_check(arg0: capsule) -> int: ... +def node_destroy(arg0: capsule) -> int: ... +def node_details(arg0: capsule) -> str: ... +def node_input_signals_max_cnt(arg0: capsule) -> int: ... +def node_is_enabled(arg0: capsule) -> bool: ... +def node_is_valid_name(arg0: str) -> bool: ... +def node_name(arg0: capsule) -> str: ... +def node_name_full(arg0: capsule) -> str: ... +def node_name_short(arg0: capsule) -> str: ... +def node_netem_fds(arg0: capsule, arg1: int) -> int: ... +def node_new(arg0: str, arg1: str) -> capsule: ... +def node_output_signals_max_cnt(arg0: capsule) -> int: ... +def node_pause(arg0: capsule) -> int: ... +def node_poll_fds(arg0: capsule, arg1: int) -> int: ... +def node_prepare(arg0: capsule) -> int: ... +@overload +def node_read(arg0: capsule, arg1: Array, arg2: int) -> int: ... +@overload +def node_read(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] +def node_restart(arg0: capsule) -> int: ... +def node_resume(arg0: capsule) -> int: ... +def node_reverse(arg0: capsule) -> int: ... +def node_start(arg0: capsule) -> int: ... +def node_stop(arg0: capsule) -> int: ... +def node_to_json_str(arg0: capsule) -> str: ... +@overload +def node_write(arg0: capsule, arg1: Array, arg2: int) -> int: ... +@overload +def node_write(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] +def sample_alloc(arg0: int) -> capsule: ... +def sample_decref(arg0: capsule) -> None: ... +def sample_length(arg0: capsule) -> int: ... +def sample_pack( + arg0: int, arg1: timespec, arg2: timespec, arg3: int, arg4: float +) -> capsule: ... +def sample_unpack( + arg0: capsule, + arg1: int, + arg2: timespec, + arg3: timespec, + arg4: int, + arg5: int, + arg6: float, +) -> None: ... +def smps_array(arg0: int) -> Array: ... diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 8ae93db1c..ca93223da 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -22,7 +22,7 @@ add_custom_target(run-python-integration-tests COMMAND python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS - python-binding + python_binding USES_TERMINAL ) diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py index 8f278651b..fbb2759ff 100644 --- a/tests/integration/python/test_binding_wrapper.py +++ b/tests/integration/python/test_binding_wrapper.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import binding as vn +import villas.node.binding as b class BindingWrapperIntegrationTests(unittest.TestCase): @@ -16,61 +16,61 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = vn.node_new(self.config, self.node_uuid) + self.test_node = b.node_new(self.config, self.node_uuid) except Exception as e: self.fail(f"new_node err: {e}") def tearDown(self): try: - vn.node_stop(self.test_node) - vn.node_destroy(self.test_node) + b.node_stop(self.test_node) + b.node_destroy(self.test_node) except Exception as e: self.fail(f"node cleanup error: {e}") def test_activity_changes(self): try: - vn.node_check(self.test_node) - vn.node_prepare(self.test_node) + b.node_check(self.test_node) + b.node_prepare(self.test_node) # starting twice - self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, b.node_start(self.test_node)) # check if the node is running - self.assertTrue(vn.node_is_enabled(self.test_node)) + self.assertTrue(b.node_is_enabled(self.test_node)) # pausing twice - self.assertEqual(0, vn.node_pause(self.test_node)) - self.assertEqual(-1, vn.node_pause(self.test_node)) + self.assertEqual(0, b.node_pause(self.test_node)) + self.assertEqual(-1, b.node_pause(self.test_node)) # resuming - self.assertEqual(0, vn.node_resume(self.test_node)) + self.assertEqual(0, b.node_resume(self.test_node)) # stopping twice - self.assertEqual(0, vn.node_stop(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, b.node_stop(self.test_node)) + self.assertEqual(0, b.node_stop(self.test_node)) # restarting - vn.node_restart(self.test_node) + b.node_restart(self.test_node) # check if everything still works after restarting - vn.node_pause(self.test_node) - vn.node_resume(self.test_node) - vn.node_stop(self.test_node) - vn.node_start(self.test_node) + b.node_pause(self.test_node) + b.node_resume(self.test_node) + b.node_stop(self.test_node) + b.node_start(self.test_node) except Exception as e: self.fail(f" err: {e}") def test_reverse_node(self): try: - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, b.node_reverse(self.test_node)) # input and output hooks/details are not reversed # input and output are reversed, can be seen with wireshark and # function test_rw_socket_and_reverse() below - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"Reversing node in and output failed: {e}") @@ -80,23 +80,23 @@ def test_reverse_node(self): # uuid can not match def test_config_from_string(self): try: - config_str = vn.node_to_json_str(self.test_node) + config_str = b.node_to_json_str(self.test_node) config_obj = json.loads(config_str) config_copy_str = json.dumps(config_obj, indent=2) - test_node = vn.node_new(config_copy_str) + test_node = b.node_new(config_copy_str) self.assertEqual( re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - vn.node_name_full(test_node), + b.node_name_full(test_node), ), re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - vn.node_name_full(self.test_node), + b.node_name_full(self.test_node), ), ) except Exception as e: @@ -113,65 +113,53 @@ def test_rw_socket_and_reverse(self): config = json.dumps(obj, indent=2) id = str(uuid.uuid4()) - test_nodes[name] = vn.node_new(config, id) + test_nodes[name] = b.node_new(config, id) for node in test_nodes.values(): - if vn.node_check(node): + if b.node_check(node): raise RuntimeError("Failed to verify node configuration") - if vn.node_prepare(node): + if b.node_prepare(node): raise RuntimeError( - f"Failed to verify {vn.node_name(node)} node config" + f"Failed to verify {b.node_name(node)} node config" ) - vn.node_start(node) + b.node_start(node) # Arrays to store samples - send_smpls = vn.SamplesArray(1) - intmdt_smpls = vn.SamplesArray(100) - recv_smpls = vn.SamplesArray(100) + send_smpls = b.SamplesArray(1) + intmdt_smpls = b.SamplesArray(100) + recv_smpls = b.SamplesArray(100) for i in range(100): # Generate signals and send over send_socket self.assertEqual( - vn.node_read( - test_nodes["signal_generator"], send_smpls, 2, 1 - ), + b.node_read(test_nodes["signal_generator"], send_smpls, 2, 1), 1, ) self.assertEqual( - vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + b.node_write(test_nodes["send_socket"], send_smpls, 1), 1 ) # read received signals and send them to recv_socket self.assertEqual( - vn.node_read( - test_nodes["intmdt_socket"], intmdt_smpls, 2, 100 - ), + b.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 2, 100), 100, ) self.assertEqual( - vn.node_write( - test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50 - ), + b.node_write(test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50), 50, ) self.assertEqual( - vn.node_write( - test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50 - ), + b.node_write(test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50), 50, ) # confirm rev_socket signals self.assertEqual( - vn.node_read( - test_nodes["recv_socket"], recv_smpls[0:50], 2, 50 - ), + b.node_read(test_nodes["recv_socket"], recv_smpls[0:50], 2, 50), 50, ) self.assertEqual( - vn.node_read( - test_nodes["recv_socket"], recv_smpls[50:100], 2, 50 - ), + b.node_read(test_nodes["recv_socket"], recv_smpls[50:100], 2, 50), 50, ) @@ -181,29 +169,27 @@ def test_rw_socket_and_reverse(self): # this can be confirmed when observing network traffic # node details do not represent this properly as of now for node in test_nodes.values(): - vn.node_reverse(node) - vn.node_stop(node) + b.node_reverse(node) + b.node_stop(node) for node in test_nodes.values(): - vn.node_start(node) + b.node_start(node) # if another 50 samples have not been allocated, # sending 100 at once is impossible with recv_smpls self.assertEqual( - vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + b.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 ) # try writing as full slice self.assertEqual( - vn.node_write( - test_nodes["intmdt_socket"], recv_smpls[0:100], 100 - ), + b.node_write(test_nodes["intmdt_socket"], recv_smpls[0:100], 100), 100, ) # cleanup for node in test_nodes.values(): - vn.node_stop(node) - vn.node_destroy(node) + b.node_stop(node) + b.node_destroy(node) except Exception as e: self.fail(f" err: {e}") @@ -216,9 +202,7 @@ def test_rw_socket_and_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [ - {"name": "tap_position", "type": "integer", "init": 0} - ], + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], }, "out": {"address": "127.0.0.1:12001"}, } @@ -276,7 +260,7 @@ def test_rw_socket_and_reverse(self): "multicast": {"enabled": False}, }, }, - "signal_gen": { + "signal_generator": { "type": "signal.v2", "limit": 100, "rate": 10, diff --git a/tests/integration/python/test_python_binding.py b/tests/integration/python/test_python_binding.py index 662f09d3f..bb20c3550 100644 --- a/tests/integration/python/test_python_binding.py +++ b/tests/integration/python/test_python_binding.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import villas_node as vn +import villas.node.python_binding as pb class BindingIntegrationTests(unittest.TestCase): @@ -16,61 +16,61 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = vn.node_new(self.config, self.node_uuid) + self.test_node = pb.node_new(self.config, self.node_uuid) except Exception as e: self.fail(f"new_node err: {e}") def tearDown(self): try: - vn.node_stop(self.test_node) - vn.node_destroy(self.test_node) + pb.node_stop(self.test_node) + pb.node_destroy(self.test_node) except Exception as e: self.fail(f"node cleanup error: {e}") def test_activity_changes(self): try: - vn.node_check(self.test_node) - vn.node_prepare(self.test_node) + pb.node_check(self.test_node) + pb.node_prepare(self.test_node) # starting twice - self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, pb.node_start(self.test_node)) # check if the node is running - self.assertTrue(vn.node_is_enabled(self.test_node)) + self.assertTrue(pb.node_is_enabled(self.test_node)) # pausing twice - self.assertEqual(0, vn.node_pause(self.test_node)) - self.assertEqual(-1, vn.node_pause(self.test_node)) + self.assertEqual(0, pb.node_pause(self.test_node)) + self.assertEqual(-1, pb.node_pause(self.test_node)) # resuming - self.assertEqual(0, vn.node_resume(self.test_node)) + self.assertEqual(0, pb.node_resume(self.test_node)) # stopping twice - self.assertEqual(0, vn.node_stop(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) # restarting - vn.node_restart(self.test_node) + pb.node_restart(self.test_node) # check if everything still works after restarting - vn.node_pause(self.test_node) - vn.node_resume(self.test_node) - vn.node_stop(self.test_node) - vn.node_start(self.test_node) + pb.node_pause(self.test_node) + pb.node_resume(self.test_node) + pb.node_stop(self.test_node) + pb.node_start(self.test_node) except Exception as e: self.fail(f" err: {e}") def test_reverse_node(self): try: - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, pb.node_reverse(self.test_node)) # input and output hooks/details are not reversed # input and output are reversed, can be seen with wireshark and # function test_rw_socket_and_reverse() below - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"Reversing node in and output failed: {e}") @@ -80,23 +80,23 @@ def test_reverse_node(self): # uuid can not match def test_config_from_string(self): try: - config_str = vn.node_to_json_str(self.test_node) + config_str = pb.node_to_json_str(self.test_node) config_obj = json.loads(config_str) config_copy_str = json.dumps(config_obj, indent=2) - test_node = vn.node_new(config_copy_str, "") + test_node = pb.node_new(config_copy_str, "") self.assertEqual( re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - vn.node_name_full(test_node), + pb.node_name_full(test_node), ), re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - vn.node_name_full(self.test_node), + pb.node_name_full(self.test_node), ), ) except Exception as e: @@ -113,54 +113,52 @@ def test_rw_socket_and_reverse(self): config = json.dumps(obj, indent=2) id = str(uuid.uuid4()) - test_nodes[name] = vn.node_new(config, id) + test_nodes[name] = pb.node_new(config, id) for node in test_nodes.values(): - if vn.node_check(node): + if pb.node_check(node): raise RuntimeError("Failed to verify node configuration") - if vn.node_prepare(node): + if pb.node_prepare(node): raise RuntimeError( - f"Failed to verify {vn.node_name(node)} node config" + f"Failed to verify {pb.node_name(node)} node config" ) - vn.node_start(node) + pb.node_start(node) # Arrays to store samples - send_smpls = vn.smps_array(1) - intmdt_smpls = vn.smps_array(100) - recv_smpls = vn.smps_array(100) + send_smpls = pb.smps_array(1) + intmdt_smpls = pb.smps_array(100) + recv_smpls = pb.smps_array(100) for i in range(100): # send_smpls holds a new sample each time, but the # old one still has a reference in the socket buffer (below) # it is necessary to allocate a new sample each time - send_smpls[0] = vn.sample_alloc(2) - intmdt_smpls[i] = vn.sample_alloc(2) - recv_smpls[i] = vn.sample_alloc(2) + send_smpls[0] = pb.sample_alloc(2) + intmdt_smpls[i] = pb.sample_alloc(2) + recv_smpls[i] = pb.sample_alloc(2) # Generate signals and send over send_socket self.assertEqual( - vn.node_read( - test_nodes["signal_generator"], send_smpls, 1 - ), + pb.node_read(test_nodes["signal_generator"], send_smpls, 1), 1, ) self.assertEqual( - vn.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + pb.node_write(test_nodes["send_socket"], send_smpls, 1), 1 ) # read received signals and send them to recv_socket self.assertEqual( - vn.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), + pb.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100, ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + pb.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100, ) # confirm rev_socket signals self.assertEqual( - vn.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 + pb.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 ) # reversing in and outputs @@ -169,24 +167,24 @@ def test_rw_socket_and_reverse(self): # this can be confirmed when observing network traffic # node details do not represent this properly as of now for node in test_nodes.values(): - vn.node_reverse(node) - vn.node_stop(node) + pb.node_reverse(node) + pb.node_stop(node) for node in test_nodes.values(): - vn.node_start(node) + pb.node_start(node) self.assertEqual( - vn.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + pb.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 ) self.assertEqual( - vn.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + pb.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), 100, ) # cleanup for node in test_nodes.values(): - vn.node_stop(node) - vn.node_destroy(node) + pb.node_stop(node) + pb.node_destroy(node) except Exception as e: self.fail(f" err: {e}") @@ -199,9 +197,7 @@ def test_rw_socket_and_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [ - {"name": "tap_position", "type": "integer", "init": 0} - ], + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], }, "out": {"address": "127.0.0.1:12001"}, } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index d42b7e6a9..44af84ea9 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -39,7 +39,7 @@ add_custom_target(run-python-unit-tests COMMAND python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS - python-binding + python_binding USES_TERMINAL ) diff --git a/tests/unit/python/test_binding_wrapper.py b/tests/unit/python/test_binding_wrapper.py index 82827e38f..781202078 100644 --- a/tests/unit/python/test_binding_wrapper.py +++ b/tests/unit/python/test_binding_wrapper.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import binding as vn +import villas.node.binding as b class BindingWrapperUnitTests(unittest.TestCase): @@ -16,26 +16,26 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = vn.node_new(self.config, self.node_uuid) + self.test_node = b.node_new(self.config, self.node_uuid) config = json.dumps(signal_test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - self.signal_test_node = vn.node_new(config, node_uuid) + self.signal_test_node = b.node_new(config, node_uuid) except Exception as e: self.fail(f"new_node err: {e}") def tearDown(self): try: - vn.node_stop(self.test_node) - vn.node_destroy(self.test_node) - vn.node_stop(self.signal_test_node) - vn.node_destroy(self.signal_test_node) + b.node_stop(self.test_node) + b.node_destroy(self.test_node) + b.node_stop(self.signal_test_node) + b.node_destroy(self.signal_test_node) except Exception as e: self.fail(f"node cleanup error: {e}") def test_start(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) - self.assertEqual(0, vn.node_start(self.signal_test_node)) + self.assertEqual(0, b.node_start(self.test_node)) + self.assertEqual(0, b.node_start(self.signal_test_node)) except Exception as e: self.fail(f"err: {e}") @@ -46,9 +46,9 @@ def test_start(self): ) def test_start_err(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, b.node_start(self.test_node)) with self.assertRaises((AssertionError, RuntimeError)): - vn.node_start(self.test_node) + b.node_start(self.test_node) except Exception as e: self.fail(f"err: {e}") @@ -56,54 +56,54 @@ def test_new(self): try: node_config = json.dumps(test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - node = vn.node_new(node_config, node_uuid) + node = b.node_new(node_config, node_uuid) self.assertIsNotNone(node) except Exception as e: self.fail(f"err: {e}") def test_check(self): try: - vn.node_check(self.test_node) + b.node_check(self.test_node) except Exception as e: self.fail(f"err: {e}") def test_prepare(self): try: - vn.node_prepare(self.test_node) + b.node_prepare(self.test_node) except Exception as e: self.fail(f"err: {e}") def test_is_enabled(self): try: - self.assertTrue(vn.node_is_enabled(self.test_node)) + self.assertTrue(b.node_is_enabled(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_pause(self): try: - self.assertEqual(-1, vn.node_pause(self.test_node)) - self.assertEqual(-1, vn.node_pause(self.test_node)) + self.assertEqual(-1, b.node_pause(self.test_node)) + self.assertEqual(-1, b.node_pause(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_resume(self): try: - self.assertEqual(0, vn.node_resume(self.test_node)) + self.assertEqual(0, b.node_resume(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_stop(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, b.node_start(self.test_node)) + self.assertEqual(0, b.node_stop(self.test_node)) + self.assertEqual(0, b.node_stop(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_restart(self): try: - self.assertEqual(0, vn.node_restart(self.test_node)) - self.assertEqual(0, vn.node_restart(self.test_node)) + self.assertEqual(0, b.node_restart(self.test_node)) + self.assertEqual(0, b.node_restart(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -112,14 +112,14 @@ def test_node_name(self): # remove color codes before checking for equality self.assertEqual( "test_node(socket)", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name(self.test_node)), + re.sub(r"\x1b\[[0-9;]*m", "", b.node_name(self.test_node)), ) except Exception as e: self.fail(f"err: {e}") def test_node_name_short(self): try: - self.assertEqual("test_node", vn.node_name_short(self.test_node)) + self.assertEqual("test_node", b.node_name_short(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -133,7 +133,7 @@ def test_node_name_full(self): + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(node)), + re.sub(r"\x1b\[[0-9;]*m", "", b.node_name_full(node)), ) except Exception as e: self.fail(f"err: {e}") @@ -144,35 +144,33 @@ def test_details(self): "layer=udp, " + "in.address=0.0.0.0:12000, " + "out.address=127.0.0.1:12001", - vn.node_details(self.test_node), + b.node_details(self.test_node), ) except Exception as e: self.fail(f"err: {e}") def test_node_to_json(self): try: - if not isinstance(vn.node_to_json(self.test_node), dict): + if not isinstance(b.node_to_json(self.test_node), dict): self.fail("Not a JSON object (dict)") except Exception as e: self.fail(f"err: {e}") def test_node_to_json_str(self): try: - print(vn.node_to_json_str(self.test_node)) - print(vn.node_to_json_str(self.test_node)) - json.loads(vn.node_to_json_str(self.test_node)) + json.loads(b.node_to_json_str(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_input_signals_max_cnt(self): try: - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_node_output_signals_max_cnt(self): try: - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -187,21 +185,21 @@ def test_node_is_valid_name(self): valid_names = ["32_characters_long_strings_valid", "valid_name"] for name in invalid_names: - self.assertFalse(vn.node_is_valid_name(name)) + self.assertFalse(b.node_is_valid_name(name)) for name in valid_names: - self.assertTrue(vn.node_is_valid_name(name)) + self.assertTrue(b.node_is_valid_name(name)) except Exception as e: self.fail(f"err: {e}") def test_reverse(self): try: # socket has reverse() implemented, expected return 0 - self.assertEqual(0, vn.node_reverse(self.test_node)) - self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, b.node_reverse(self.test_node)) + self.assertEqual(0, b.node_reverse(self.test_node)) # signal.v2 has not reverse() implemented, expected return 1 - self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) - self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) + self.assertEqual(-1, b.node_reverse(self.signal_test_node)) + self.assertEqual(-1, b.node_reverse(self.signal_test_node)) except Exception as e: self.fail(f"err: {e}") @@ -213,9 +211,7 @@ def test_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [ - {"name": "tap_position", "type": "integer", "init": 0} - ], + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], }, "out": {"address": "127.0.0.1:12001"}, } diff --git a/tests/unit/python/test_python_binding.py b/tests/unit/python/test_python_binding.py index 01ac98c6c..1ded31c73 100644 --- a/tests/unit/python/test_python_binding.py +++ b/tests/unit/python/test_python_binding.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import villas_node as vn +import villas.node.python_binding as pb class BindingUnitTests(unittest.TestCase): @@ -16,26 +16,26 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = vn.node_new(self.config, self.node_uuid) + self.test_node = pb.node_new(self.config, self.node_uuid) config = json.dumps(signal_test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - self.signal_test_node = vn.node_new(config, node_uuid) + self.signal_test_node = pb.node_new(config, node_uuid) except Exception as e: self.fail(f"new_node err: {e}") def tearDown(self): try: - vn.node_stop(self.test_node) - vn.node_destroy(self.test_node) - vn.node_stop(self.signal_test_node) - vn.node_destroy(self.signal_test_node) + pb.node_stop(self.test_node) + pb.node_destroy(self.test_node) + pb.node_stop(self.signal_test_node) + pb.node_destroy(self.signal_test_node) except Exception as e: self.fail(f"node cleanup error: {e}") def test_start(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) - self.assertEqual(0, vn.node_start(self.signal_test_node)) + self.assertEqual(0, pb.node_start(self.test_node)) + self.assertEqual(0, pb.node_start(self.signal_test_node)) except Exception as e: self.fail(f"err: {e}") @@ -48,9 +48,9 @@ def test_start(self): ) def test_start_err(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) + self.assertEqual(0, pb.node_start(self.test_node)) with self.assertRaises((AssertionError, RuntimeError)): - vn.node_start(self.test_node) + pb.node_start(self.test_node) except Exception as e: self.fail(f"err: {e}") @@ -58,54 +58,54 @@ def test_new(self): try: node_config = json.dumps(test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - node = vn.node_new(node_config, node_uuid) + node = pb.node_new(node_config, node_uuid) self.assertIsNotNone(node) except Exception as e: self.fail(f"err: {e}") def test_check(self): try: - vn.node_check(self.test_node) + pb.node_check(self.test_node) except Exception as e: self.fail(f"err: {e}") def test_prepare(self): try: - vn.node_prepare(self.test_node) + pb.node_prepare(self.test_node) except Exception as e: self.fail(f"err: {e}") def test_is_enabled(self): try: - self.assertTrue(vn.node_is_enabled(self.test_node)) + self.assertTrue(pb.node_is_enabled(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_pause(self): try: - self.assertEqual(-1, vn.node_pause(self.test_node)) - self.assertEqual(-1, vn.node_pause(self.test_node)) + self.assertEqual(-1, pb.node_pause(self.test_node)) + self.assertEqual(-1, pb.node_pause(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_resume(self): try: - self.assertEqual(0, vn.node_resume(self.test_node)) + self.assertEqual(0, pb.node_resume(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_stop(self): try: - self.assertEqual(0, vn.node_start(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) - self.assertEqual(0, vn.node_stop(self.test_node)) + self.assertEqual(0, pb.node_start(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_restart(self): try: - self.assertEqual(0, vn.node_restart(self.test_node)) - self.assertEqual(0, vn.node_restart(self.test_node)) + self.assertEqual(0, pb.node_restart(self.test_node)) + self.assertEqual(0, pb.node_restart(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -114,14 +114,14 @@ def test_node_name(self): # remove color codes before checking for equality self.assertEqual( "test_node(socket)", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name(self.test_node)), + re.sub(r"\x1b\[[0-9;]*m", "", pb.node_name(self.test_node)), ) except Exception as e: self.fail(f"err: {e}") def test_node_name_short(self): try: - self.assertEqual("test_node", vn.node_name_short(self.test_node)) + self.assertEqual("test_node", pb.node_name_short(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -135,7 +135,7 @@ def test_node_name_full(self): + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - re.sub(r"\x1b\[[0-9;]*m", "", vn.node_name_full(node)), + re.sub(r"\x1b\[[0-9;]*m", "", pb.node_name_full(node)), ) except Exception as e: self.fail(f"err: {e}") @@ -146,20 +146,20 @@ def test_details(self): "layer=udp, " + "in.address=0.0.0.0:12000, " + "out.address=127.0.0.1:12001", - vn.node_details(self.test_node), + pb.node_details(self.test_node), ) except Exception as e: self.fail(f"err: {e}") def test_input_signals_max_cnt(self): try: - self.assertEqual(1, vn.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"err: {e}") def test_node_output_signals_max_cnt(self): try: - self.assertEqual(0, vn.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) except Exception as e: self.fail(f"err: {e}") @@ -174,21 +174,21 @@ def test_node_is_valid_name(self): valid_names = ["32_characters_long_strings_valid", "valid_name"] for name in invalid_names: - self.assertFalse(vn.node_is_valid_name(name)) + self.assertFalse(pb.node_is_valid_name(name)) for name in valid_names: - self.assertFalse(vn.node_is_valid_name(name)) + self.assertTrue(pb.node_is_valid_name(name)) except Exception as e: self.fail(f"err: {e}") def test_reverse(self): try: # socket has reverse() implemented, expected return 0 - self.assertEqual(0, vn.node_reverse(self.test_node)) - self.assertEqual(0, vn.node_reverse(self.test_node)) + self.assertEqual(0, pb.node_reverse(self.test_node)) + self.assertEqual(0, pb.node_reverse(self.test_node)) # signal.v2 has not reverse() implemented, expected return 1 - self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) - self.assertEqual(-1, vn.node_reverse(self.signal_test_node)) + self.assertEqual(-1, pb.node_reverse(self.signal_test_node)) + self.assertEqual(-1, pb.node_reverse(self.signal_test_node)) except Exception as e: self.fail(f"err: {e}") @@ -200,9 +200,7 @@ def test_reverse(self): "layer": "udp", "in": { "address": "*:12000", - "signals": [ - {"name": "tap_position", "type": "integer", "init": 0} - ], + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], }, "out": {"address": "127.0.0.1:12001"}, } From f73fc67c71a828f8d31cacd3b7803edf79566017 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 12 Sep 2025 09:32:51 +0200 Subject: [PATCH 25/32] binding stubs: add missing SPDX header Also reformatted the stubs with pre-commit run on the `python:3.12.10-slim-bookworm` image that is also used in the pipeline. Running pre-commit within the Fedora dev image has different behavior. Signed-off-by: Kevin Vu te Laar --- python/villas/node/binding.pyi | 11 +++++++++-- python/villas/node/python_binding.pyi | 13 ++++++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/python/villas/node/binding.pyi b/python/villas/node/binding.pyi index 335be5e0d..995819a6a 100644 --- a/python/villas/node/binding.pyi +++ b/python/villas/node/binding.pyi @@ -1,9 +1,17 @@ +""" +@generated by mypy (partial). Manual edits applied for function decorators. +isort:skip_file + +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" + from _typeshed import Incomplete from typing import Any, Callable logger: Incomplete - class SamplesArray: def __init__(self, length: int) -> None: ... def __len__(self) -> int: ... @@ -11,7 +19,6 @@ class SamplesArray: def __copy__(self) -> None: ... def __deepcopy__(self) -> None: ... - def _warn_if_not_implemented( func: Callable[..., Any], ) -> Callable[..., Any]: ... diff --git a/python/villas/node/python_binding.pyi b/python/villas/node/python_binding.pyi index 309c61a6c..81b91d128 100644 --- a/python/villas/node/python_binding.pyi +++ b/python/villas/node/python_binding.pyi @@ -1,11 +1,19 @@ +""" +@generated by mypy (partial). Manual edits applied for pybind11 binding. +isort:skip_file + +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" + import typing -from typing import Any, Callable, overload +from typing import Any, overload Array = Any capsule = Any timespec = Any - class SamplesArray: def __init__(self, len: int) -> None: ... def bulk_alloc(self, arg0: int, arg1: int, arg2: int) -> None: ... @@ -14,7 +22,6 @@ class SamplesArray: def __iter__(self) -> typing.Iterator[capsule]: ... def __setitem__(self, arg0: int, arg1: capsule) -> None: ... - def memory_init(arg0: int) -> int: ... def node_check(arg0: capsule) -> int: ... def node_destroy(arg0: capsule) -> int: ... From 124a1f5573bc13672d939dde90babab2339c0a4c Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Fri, 12 Sep 2025 09:41:42 +0200 Subject: [PATCH 26/32] python binding: install and fix CI Install: - As of Python 3.3+, `__init__.py` is not required in site-package folders. Those folders are seen as module by default. - Helps with file ownership when installing the binding wrapper via pip as part of the villas-node package and the bindings from the codebase. CI: - Fixed wrong variable name in the CI script. - .so file should be found and symlinked properly into `/python/villas/node` for testing purposes. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 4 ++-- python/binding/CMakeLists.txt | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b0682fee..15f36afdd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,11 +181,11 @@ test:python_unit_integration: - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1) - - link_path=${PWD}/python/villas/node/$(basename $python_binding) + - link_path=${PWD}/python/villas/node/$(basename $binding_path) - ln -sf $binding_path $link_path - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests - - rm $node_path + - rm $link_path needs: - job: "build:source: [fedora]" artifacts: true diff --git a/python/binding/CMakeLists.txt b/python/binding/CMakeLists.txt index bb81221ec..d1b346da1 100644 --- a/python/binding/CMakeLists.txt +++ b/python/binding/CMakeLists.txt @@ -28,13 +28,6 @@ if(pybind11_FOUND) COMPONENT lib LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/ ) - file(WRITE "${CMAKE_BINARY_DIR}/python/villas/__init__.py" "") - file(WRITE "${CMAKE_BINARY_DIR}/python/villas/node/__init__.py" "") - install( - DIRECTORY "${CMAKE_BINARY_DIR}/python/villas" - DESTINATION "${PYTHON_SITE_PACKAGES}" - FILES_MATCHING PATTERN "__init__.py" - ) else() message(STATUS "pybind11 not found. Skipping Python wrapper build.") endif() From 1af1e49ceb64e3eb790b2056750e0a3948cdaee4 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 15 Dec 2025 17:31:35 +0100 Subject: [PATCH 27/32] Rework Binding Wrapper Binding rework to a class design. Node: - Stores SamplesArray, Config, cpp-Node handle. - Automatically deals with SamplesArray sizing and (de-)allocation of samples. - Removed `node_netem_fds()` and `node_poll_fds`. - `sample_pack()` and `sample_unpack` can only read/write samples to a SamplesArray within a node. - Added `sample_details()` for inspecting sample content. - Added `_SamplesArray` helper class to enable more pythonic syntax. node_capi.cpp, node.h: - Aligned `sequence` parameter type in `sample_pack()` and `sample_unpack()` with `sequence` field of `struct Sample` (uint64_t) Signed-off-by: Kevin Vu te Laar --- include/villas/node.h | 5 +- lib/node_capi.cpp | 6 +- python/binding/capi_python_binding.cpp | 212 ++++++-- python/villas/node/binding.py | 709 +++++++++++++++---------- 4 files changed, 611 insertions(+), 321 deletions(-) diff --git a/include/villas/node.h b/include/villas/node.h index 4ba8ed6f4..ba0e3d3be 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -6,6 +6,7 @@ */ #pragma once +#include #include #include @@ -67,11 +68,11 @@ unsigned sample_length(vsample *smp); void sample_decref(vsample *smp); -vsample *sample_pack(unsigned seq, struct timespec *ts_origin, +vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, unsigned len, double *values); -void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin, +void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, int *flags, unsigned *len, double *values); diff --git a/lib/node_capi.cpp b/lib/node_capi.cpp index 92ee338e5..97d377482 100644 --- a/lib/node_capi.cpp +++ b/lib/node_capi.cpp @@ -152,12 +152,12 @@ unsigned sample_length(vsample *s) { return smp->length; } -vsample *sample_pack(unsigned seq, struct timespec *ts_origin, +vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, unsigned len, double *values) { auto *smp = sample_alloc_mem(len); - smp->sequence = seq; + smp->sequence = *seq; smp->ts.origin = *ts_origin; smp->ts.received = *ts_received; smp->length = len; @@ -169,7 +169,7 @@ vsample *sample_pack(unsigned seq, struct timespec *ts_origin, return (vsample *)smp; } -void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin, +void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, int *flags, unsigned *len, double *values) { auto *smp = (Sample *)s; diff --git a/python/binding/capi_python_binding.cpp b/python/binding/capi_python_binding.cpp index f5271b237..680e0f8af 100644 --- a/python/binding/capi_python_binding.cpp +++ b/python/binding/capi_python_binding.cpp @@ -5,12 +5,20 @@ * Systems, RWTH Aachen University SPDX-License-Identifier: Apache-2.0 */ +#include +#include +#include + #include #include +#include +#include +#include #include #include #include +#include extern "C" { #include @@ -18,19 +26,23 @@ extern "C" { namespace py = pybind11; -class Array { +class SamplesArray { public: - Array(unsigned int len) { - smps = new vsample *[len](); + SamplesArray(unsigned int len = 0) { + smps = (len > 0) ? new vsample *[len]() : nullptr; this->len = len; } - Array(const Array &) = delete; - Array &operator=(const Array &) = delete; + SamplesArray(const SamplesArray &) = delete; + SamplesArray &operator=(const SamplesArray &) = delete; - ~Array() { + ~SamplesArray() { + if (!smps) + return; for (unsigned int i = 0; i < len; ++i) { - sample_decref(smps[i]); - smps[i] = nullptr; + if (smps[i]) { + sample_decref(smps[i]); + smps[i] = nullptr; + } } delete[] smps; } @@ -47,9 +59,45 @@ class Array { } } - vsample *&operator[](unsigned int idx) { return smps[idx]; } + /* + * Performs a resize of the underlying SamplesArray copying each Sample. + * Shrinking has asymmetric behavior which may be undesired. + * Therefore use clear(). + */ + int grow(unsigned int add) { + unsigned int new_len = this->len + add; + vsample **smps_new = new vsample *[new_len](); + for (unsigned int i = 0; i < this->len; ++i) { + smps_new[i] = smps[i]; + } + delete[] smps; + this->smps = smps_new; + this->len = new_len; + + return new_len; + } + + int clear() { + if (this->smps) { + unsigned int i = 0; + for (; i < len; ++i) { + sample_decref(smps[i]); + smps[i] = nullptr; + } + delete[] smps; + smps = nullptr; + this->len = 0; + return i; + } + return -1; + } + + vsample *&operator[](unsigned int idx) { + vsample *&ref = smps[idx]; + return ref; + } - vsample *&operator[](unsigned int idx) const { return smps[idx]; } + vsample *operator[](unsigned int idx) const { return smps[idx]; } vsample **get_smps() { return smps; } @@ -60,8 +108,15 @@ class Array { unsigned int len; }; +struct timespec ns_to_timespec(int64_t time_ns) { + struct timespec ts; + ts.tv_nsec = time_ns / 1'000'000'000LL; + ts.tv_sec = time_ns % 1'000'000'000LL; + return ts; +} + /* pybind11 can not deal with (void **) as function input parameters, - * therefore we have to cast a simple (void *) pointer to the corresponding type + * therefore cast a simple (void *) pointer to the corresponding type * * wrapper bindings, sorted alphabetically * @param villas_node Name of the module to be bound @@ -105,10 +160,6 @@ PYBIND11_MODULE(python_binding, m) { [](void *n) -> const char * { return node_name_short((vnode *)n); }, py::return_value_policy::copy); - m.def("node_netem_fds", [](void *n, int fds[]) -> int { - return node_netem_fds((vnode *)n, fds); - }); - m.def( "node_new", [](const char *json_str, const char *id_str) -> vnode * { @@ -132,7 +183,7 @@ PYBIND11_MODULE(python_binding, m) { return ret; } }, - py::return_value_policy::reference); + py::return_value_policy::take_ownership); m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { return node_output_signals_max_cnt((vnode *)n); @@ -140,14 +191,10 @@ PYBIND11_MODULE(python_binding, m) { m.def("node_pause", [](void *n) -> int { return node_pause((vnode *)n); }); - m.def("node_poll_fds", [](void *n, int fds[]) -> int { - return node_poll_fds((vnode *)n, fds); - }); - m.def("node_prepare", [](void *n) -> int { return node_prepare((vsample *)n); }); - m.def("node_read", [](void *n, Array &a, unsigned cnt) -> int { + m.def("node_read", [](void *n, SamplesArray &a, unsigned cnt) -> int { return node_read((vnode *)n, a.get_smps(), cnt); }); @@ -178,7 +225,7 @@ PYBIND11_MODULE(python_binding, m) { return py_str; }); - m.def("node_write", [](void *n, Array &a, unsigned cnt) -> int { + m.def("node_write", [](void *n, SamplesArray &a, unsigned cnt) -> int { return node_write((vnode *)n, a.get_smps(), cnt); }); @@ -187,7 +234,8 @@ PYBIND11_MODULE(python_binding, m) { }); m.def( - "smps_array", [](unsigned int len) -> Array * { return new Array(len); }, + "smps_array", + [](unsigned int len) -> SamplesArray * { return new SamplesArray(len); }, py::return_value_policy::take_ownership); m.def("sample_alloc", @@ -199,36 +247,128 @@ PYBIND11_MODULE(python_binding, m) { sample_decref(*smp); }); - m.def("sample_length", - [](void *smp) -> unsigned { return sample_length((vsample *)smp); }); + m.def("sample_length", [](void *smp) -> unsigned { + if (smp) { + return sample_length((vsample *)smp); + } else { + return -1; + } + }); - m.def("sample_pack", &sample_pack, py::return_value_policy::reference); + m.def( + "sample_pack", + [](void *s, std::optional ts_origin_ns, + std::optional ts_received_ns) -> vsample * { + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); + + auto smp = (villas::node::Sample *)s; + uint64_t *seq = &smp->sequence; + unsigned len = smp->length; + double *values = (double *)smp->data; + + return sample_pack(seq, &ts_origin, &ts_received, len, values); + }, + py::return_value_policy::reference); + + m.def( + "sample_pack", + [](const py::list values, std::optional ts_origin_ns, + std::optional ts_received_ns, unsigned seq = 0) -> void * { + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); + + unsigned values_len = values.size(); + double cvalues[values.size()]; + for (unsigned int i = 0; i < values_len; ++i) { + cvalues[i] = values[i].cast(); + } + uint64_t sequence = seq; + + auto tmp = (void *)sample_pack(&sequence, &ts_origin, &ts_received, + values_len, cvalues); + return tmp; + }, + py::return_value_policy::reference); m.def( "sample_unpack", - [](void *smp, unsigned *seq, struct timespec *ts_origin, - struct timespec *ts_received, int *flags, unsigned *len, - double *values) -> void { - return sample_unpack((vsample *)smp, seq, ts_origin, ts_received, flags, - len, values); + [](void *ss, void *ds, std::optional ts_origin_ns, + std::optional ts_received_ns) -> void { + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); + + auto srcSmp = (villas::node::Sample **)ss; + auto destSmp = (villas::node::Sample **)ds; + + if (!*srcSmp) { + throw std::runtime_error("Tried to unpack empty sample!"); + } + if (!*destSmp) { + *destSmp = (villas::node::Sample *)sample_alloc((*srcSmp)->length); + } else if ((*destSmp)->capacity < (*srcSmp)->length) { + sample_decref(*(vsample **)destSmp); + *destSmp = (villas::node::Sample *)sample_alloc((*srcSmp)->length); + } + + uint64_t *seq = &(*destSmp)->sequence; + int *flags = &(*destSmp)->flags; + unsigned *len = &(*destSmp)->length; + double *values = (double *)(*destSmp)->data; + + sample_unpack(*(vsample **)srcSmp, seq, &ts_origin, &ts_received, flags, + len, values); }, py::return_value_policy::reference); - py::class_(m, "SamplesArray") + m.def("sample_details", [](void *s) { + auto smp = (villas::node::Sample *)s; + if (!smp) { + return py::dict(); + } + + py::dict d; + d["sequence"] = smp->sequence; + d["length"] = smp->length; + d["capacity"] = smp->capacity; + d["flags"] = smp->flags; + d["refcnt"] = smp->refcnt.load(); + d["ts_origin"] = time_to_double(&smp->ts.origin); + d["ts_received"] = time_to_double(&smp->ts.received); + + py::list data; + for (unsigned int i = 0; i < smp->length; ++i) { + data.append((double)smp->data[i]); + } + d["data"] = data; + + return d; + }); + + py::class_(m, "SamplesArray") .def(py::init(), py::arg("len")) .def("__getitem__", - [](Array &a, unsigned int idx) { + [](SamplesArray &a, unsigned int idx) { assert(idx < a.size() && "Index out of bounds"); return a[idx]; }) .def("__setitem__", - [](Array &a, unsigned int idx, void *smp) { + [](SamplesArray &a, unsigned int idx, void *smp) { assert(idx < a.size() && "Index out of bounds"); if (a[idx]) { sample_decref(a[idx]); } a[idx] = (vsample *)smp; }) - .def("get_block", &Array::get_block) - .def("bulk_alloc", &Array::bulk_alloc); + .def("__len__", &SamplesArray::size) + .def("bulk_alloc", &SamplesArray::bulk_alloc) + .def("grow", &SamplesArray::grow) + .def("get_block", &SamplesArray::get_block) + .def("clear", &SamplesArray::clear); } diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py index 7556426d0..d9b47b282 100644 --- a/python/villas/node/binding.py +++ b/python/villas/node/binding.py @@ -4,378 +4,527 @@ SPDX-License-Identifier: Apache-2.0 """ # noqa: E501 -from typing import Optional, Union +from typing import Any, Optional, Union import functools import json import logging +import weakref import villas.node.python_binding as vn +Capsule = Any logger = logging.getLogger("villas.node") +# helper functions -class SamplesArray: - """ - Wrapper for a block of samples with automatic memory management. - Supports: - - Reading block slices in combination with `node_read()`. - - Writing block slices in combination with `node_write()`. - - Automatic (de-)allocation of samples. +# function decorator for optional node_compat function calls +# that would return -1 if a function is not implemented +def _warn_if_not_implemented(func): + """ + Decorator to warn if specific `node_*()` functions are not implemented. - Notes: - - Block slices are a slices with step size 1 + Returns: + Wrapping function that logs a warning if the return value is -1. """ - def _bulk_allocation(self, start_idx: int, end_idx: int, smpl_length: int): - """ - Allocates a block of samples. + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + ret = func(self, *args, **kwargs) + if ret == -1 and hasattr(self, "_hndle") and self._hndle is not None: + msg = ( + f"[\033[93mWarning\033[0m]: Function '{func.__name__}()' " + + "is not implemented for node type " + + f"'{vn.node_name(self._hndle)}'." + ) + logger.warning(msg) + return ret - Args: - start_idx (int): Starting Index of a block. - end_idx (int): One past the last index to allocate. - smpl_length (int): length of sample per slot + return wrapper - Notes: - - Block is determined by `start_idx`, `end_idx` - - Samples can hold multiple signals. - - `smpl_length` corresponds to the number of signals. + +# node API bindings + + +class Node: + class _SampleSlice: + """ + Helper class to achieve overloaded functions. + Nodes accessed via the `[]` operator, it always returns a _SampleSlice. """ - return self._smps.bulk_alloc(start_idx, end_idx, smpl_length) - def _get_block_handle(self, start_idx: Optional[int] = None): + def __init__(self, node, idx): + self.node = weakref.proxy(node) + self.idx = idx + + def details(self): + return self.node.sample_details(self.idx) + + def read_from(self, sample_length, count=None): + return self.node.read_from(sample_length, count, self.idx) + + def write_to(self, node, count=None): + return self.node.write_to(node, count, self.idx) + + def pack_from( + self, + values: Union[float, list[float], Capsule], + ts_origin: Optional[int] = None, + ts_received: Optional[int] = None, + seq: int = 0, + ): + if isinstance(values, self.__class__): + return self.node.pack_from( + self.idx, + values.node._smps[values.idx], + ts_origin, + ts_received, + seq, + ) + else: + return self.node.pack_from( + self.idx, values, ts_origin, ts_received, seq + ) + + def unpack_to( + self, + target: Capsule, + ts_origin: Optional[int] = None, + ts_received: Optional[int] = None, + ): + if isinstance(target, self.__class__): + return self.node.unpack_to( + self.idx, + target.node, + target.idx, + ts_origin, + ts_received, + ) + else: + raise ValueError( + "The destination must be an existing Node with an index!" + ) + + # helper functions + @staticmethod + def _ensure_capacity(smps, cap: int): """ - Get a handle to a block of samples. + Resize SamplesArray if its capacity is less than the desired capacity. Args: - start_idx (Optional[int]): Starting index of the block. Default 0. + smps: SamplesArray stored in the Node + cap (int): Desired capacity of the SamplesArray + """ + smp_cap = len(smps) + if smp_cap < cap: + smps.grow(cap - smp_cap) + + @staticmethod + def _resolve_range( + start: Optional[int], stop: Optional[int], count: Optional[int] + ) -> tuple[int, int, int]: + """ + Resolve a range dependent on start, stop and count. + At least two must be provided. + + Args: + start (int): Desired start index + stop (int): Desired stop index + count (int): Desired span of the range Returns: - vsample**: Handle to the underlying block masked as `void *`. + tuple(start, stop, count) """ - if start_idx is None: - return self._smps.get_block(0) - else: - return self._smps.get_block(start_idx) - - def __init__(self, length: int): + provided = sum(i is not None for i in (start, stop, count)) + if provided == 1: + raise ValueError("Two of start, stop, count must be provided") + elif provided == 2: + if start is None: + start = stop - count + if start < 0: + raise ValueError("Negative start index") + elif stop is None: + stop = start + count + else: + count = stop - start + return start, stop, count + + def __init__(self, config, uuid: Optional[str] = None, size=0): """ - Initialize a SamplesArray. + Initiallize a new node from config. Notes: - - Each sample slot can hold one sample allocated via `node_read()`. - - Sample can contain multiple signals, depending on `smpl_length`. + - Capsule is available via self._hndle. + - Capsule is available via self.config. """ - self._smps = vn.smps_array(length) - self._len = length + self.config = config + self._smps = vn.smps_array(size) + if uuid is None: + self._hndle = vn.node_new(config, "") + else: + self._hndle = vn.node_new(config, uuid) - def __len__(self): + def __del__(self): """ - Returns the length of the SamplesArray. - Corresponds to the amount of samples it holds. + Stop and delete a node if the class object is deleted. """ - return self._len + vn.node_stop(self._hndle) + vn.node_destroy(self._hndle) def __getitem__(self, idx: Union[int, slice]): """Return tuple containing self and index/slice for node operations.""" - if isinstance(idx, slice): - return (self, idx) - elif isinstance(idx, int): - return (self, idx) + if isinstance(idx, (int, slice)): + return Node._SampleSlice(self, idx) else: logger.warning("Improper array index") raise ValueError("Improper Index") + def __setitem__(self, obj): + if isinstance(obj, Node): + self.__del__() + self.config = obj.config + self._smps = obj._smps + else: + raise RuntimeError(f"{obj} is not of type `Node`") + + def __len__(self): + return len(self._smps) + def __copy__(self): """Disallow shallow copying.""" - raise RuntimeError("Copying SamplesArray is not allowed") + raise RuntimeError("Copying Node is not allowed") def __deepcopy__(self): - """Disallow deep copying.""" - raise RuntimeError("Copying SamplesArray is not allowed") - + """Disallow deep copying""" + raise RuntimeError("Copying a Node is not allowed") -# helper functions - - -# function decorator for optional node_compat function calls -# that would return -1 if a function is not implemented -def _warn_if_not_implemented(func): - """ - Decorator to warn if specific `node_*()` functions are not implemented. - - Returns: - Wrapping function that logs a warning if the return value is -1. - """ - - @functools.wraps(func) - def wrapper(*args, **kwargs): - ret = func(*args, **kwargs) - if ret == -1: - msg = f"[\033[33mWarning\033[0m]: Function '{func.__name__}()' \ - is not implemented for node type '{vn.node_name(*args)}'." - logger.warning(msg) - return ret + # bindings + @staticmethod + def memory_init(hugepages: int): + """ + Initialize internal VILLASnode memory system. - return wrapper + Args: + hugepages (int): Amount of hugepages to be used. + Notes: + - Should be called once before any memory allocation is done. + - Falls back to mmap if hugepages or root privilege unavailable. + """ + return vn.memory_init(hugepages) + + def check(self): + """Check node.""" + return vn.node_check(self._hndle) + + def details(self): + """Get node details.""" + return vn.node_details(self._hndle) + + def input_signals_max_cnt(self): + """Get max input signal count.""" + return vn.node_input_signals_max_cnt(self._hndle) + + def is_enabled(self): + """Check whether or not node is enabled.""" + return vn.node_is_enabled(self._hndle) + + @staticmethod + def is_valid_name(name: str): + """Check if a name can be used for a node.""" + return vn.node_is_valid_name(name) + + def name(self): + """Get node name.""" + return vn.node_name(self._hndle) + + def name_full(self): + """Get node name with full details.""" + return vn.node_name_full(self._hndle) + + def name_short(self): + """Get node name with less details.""" + return vn.node_name_short(self._hndle) + + def output_signals_max_cnt(self): + """Get max output signal count.""" + return vn.node_output_signals_max_cnt(self._hndle) + + def pause(self): + """Pause a node""" + return vn.node_pause(self._hndle) + + def prepare(self): + """Prepare a node""" + return vn.node_prepare(self._hndle) + + @_warn_if_not_implemented + def read_from( + self, + sample_length: int, + cnt: Optional[int] = None, + idx=None, + ): + """ + Read samples from a node into SamplesArray or block slice of samples. -# node API bindings + Args: + sample_length (int): Length of each sample (number of signals). + cnt (int): Number of samples to read. + Returns: + int: Number of samples read on success or -1. -def memory_init(*args): - return vn.memory_init(*args) + Notes: + - Return value may vary depending on node type. + - This function may be blocking. + """ + if idx is None: + if cnt is None: + raise ValueError("Count is None") + # resize _smps if too small + Node._ensure_capacity(self._smps, cnt) -def node_check(node): - """Check node.""" - return vn.node_check(node) + # allocate new samples + self._smps.bulk_alloc(0, len(self._smps), sample_length) + return vn.node_read(self._hndle, self._smps.get_block(0), cnt) -def node_destroy(node): - """ - Delete a node. + if isinstance(idx, int): + if cnt is None: + raise ValueError("Count is None") + start = idx + stop = start + cnt - Notes: - - Note should be stopped first. - """ - return vn.node_destroy(node) + # if too small, resize _smps to match stop index + Node._ensure_capacity(self._smps, stop) + # allocate new samples + self._smps.bulk_alloc(start, stop, sample_length) -def node_details(node): - """Get node details.""" - return vn.node_details(node) + # read onward from index start + return vn.node_read(self._hndle, self._smps.get_block(start), cnt) + elif isinstance(idx, slice): + start, stop, cnt = Node._resolve_range(idx.start, idx.stop, cnt) -def node_input_signals_max_cnt(node): - """Get max input signal count.""" - return vn.node_input_signals_max_cnt(node) + # check for length mismatch + if (stop - start) != cnt: + raise ValueError("Slice length and sample count do not match!") + # if too small, resize _smps to match stop index + Node._ensure_capacity(self._smps, stop) + # allocate new samples + self._smps.bulk_alloc(start, stop, sample_length) -def node_is_enabled(node): - """Check whether or not node is enabled.""" - return vn.node_is_enabled(node) + # read onward from index start + return vn.node_read(self._hndle, self._smps.get_block(start), cnt) + else: + logger.warning("Invalid samples Parameter") + return -1 -def node_is_valid_name(name: str): - """Check if a name can be used for a node.""" - return vn.node_is_valid_name(name) + def restart(self): + """Restart a node.""" + return vn.node_restart(self._hndle) + def resume(self): + """Resume a node.""" + return vn.node_resume(self._hndle) -def node_name(node): - """Get node name.""" - return vn.node_name(node) + @_warn_if_not_implemented + def reverse(self): + """ + Reverse node input and output. + Notes: + - Hooks are not reversed. + - Some nodes should be stopped or restarted before reversing. + - Nodes with in-/output buffers should be stopped before reversing. + """ + return vn.node_reverse(self._hndle) -def node_name_full(node): - """Get node name with full details.""" - return vn.node_name_full(node) + def start(self): + """ + Start a node. + Notes: + - Nodes are not meant to be started again without stopping first. + """ + return vn.node_start(self._hndle) -def node_name_short(node): - """Get node name with less details.""" - return vn.node_name_short(node) + def stop(self): + """ + Stop a node. + Notes: + - Use before starting a node again. + - May delete in-/output buffers of a node. + """ + return vn.node_stop(self._hndle) -def node_netem_fds(*args): - return vn.node_netem_fds(*args) + def to_json(self): + """ + Return the node configuration as json object. + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + json_str = vn.node_to_json_str(self._hndle) + json_obj = json.loads(json_str) + return json_obj -def node_new(config, uuid: Optional[str] = None): - """ - Create a new node. + def to_json_str(self): + """ + Returns the node configuration as string. - Args: - config (json/str): Configuration of the node. - uuid (Optional[str]): Unique identifier of the node. - If `None`, VILLASnode will assign one by default. + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + return vn.node_to_json_str(self._hndle) - Returns: - vnode *: Handle to a node. - """ - if uuid is None: - return vn.node_new(config, "") - else: - return vn.node_new(config, uuid) + @_warn_if_not_implemented + def write_to(self, node, cnt: Optional[int] = None, idx=None): + """ + Write samples from a SamplesArray fully or as block slice into a node. + Args: + node: Node handle. + cnt (int): Number of samples to write. -def node_output_signals_max_cnt(node): - return vn.node_output_signals_max_cnt(node) + Returns: + int: Number of samples written on success or -1. + Notes: + - Return value may vary depending on node type. + """ + if idx is None: + if cnt is None: + raise ValueError("Count is None") -def node_pause(node): - """Pause a node""" - return vn.node_pause(node) + return vn.node_write(self._hndle, node._smps.get_block(0), cnt) + if isinstance(idx, int): + if cnt is None: + raise ValueError("Count is None") -def node_poll_fds(*args): - return vn.node_poll_fds(*args) + start = idx + stop = start + cnt + return vn.node_write(self._hndle, node._smps.get_block(start), cnt) -def node_prepare(node): - """Prepare a node""" - return vn.node_prepare(node) + if isinstance(idx, slice): + start, stop, _ = idx.indices(len(self._smps)) + if cnt is None: + cnt = stop - start -@_warn_if_not_implemented -def node_read(node, samples, sample_length, count): - """ - Read samples from a node into SamplesArray or a block slice of its samples. + print(start, stop, cnt, len(self._smps)) - Args: - node: Node handle. - samples: Either a SamplesArray or a tuple (SamplesArray, index/slice). - sample_length: Length of each sample (number of signals). - count: Number of samples to read. + # check for length mismatch + if (stop - start) != cnt: + raise ValueError("Slice length and sample count do not match.") + # check for out of bounds + if stop > len(self._smps): + raise IndexError("Out of bounds") - Returns: - int: Number of samples read on success or -1 if not implemented. + return vn.node_write(self._hndle, node._smps.get_block(start), cnt) - Notes: - - Return value may vary depending on node type. - - This function may be blocking. - """ - if isinstance(samples, SamplesArray): - samples._bulk_allocation(0, len(samples), sample_length) - return vn.node_read(node, samples._get_block_handle(0), count) - elif isinstance(samples, tuple): - smpls = samples[0] - if not isinstance(samples[1], slice): - raise ValueError("Invalid samples Parameter") - start, stop, _ = samples[1].indices(len(smpls)) - - # check for length mismatch - if (stop - start) != count: - raise ValueError("Slice length and sample count do not match.") - # check if out of bounds - if stop > len(smpls): - raise IndexError("Out of bounds") - - # allocate new samples and get block handle - samples[0]._bulk_allocation(start, stop, sample_length) - handle = samples[0]._get_block_handle(start) - - return vn.node_read(node, handle, count) - else: logger.warning("Invalid samples Parameter") return -1 + def sample_length(self, idx: int): + """Get the length of a sample.""" + if 0 <= idx and idx < len(self._smps): + return vn.sample_length(self._smps[idx]) + else: + raise IndexError(f"No Sample at index: {idx}") + + def pack_from( + self, + idx: int, + values: Union[float, list[float], Capsule], + ts_orig: Optional[int] = None, + ts_recv: Optional[int] = None, + seq: int = 0, + ): + """ + Packs a given sample from a source sample or value list. -def node_restart(node): - """Restart a node.""" - return vn.node_restart(node) - - -def node_resume(node): - """Resume a node.""" - return vn.node_resume(node) - - -@_warn_if_not_implemented -def node_reverse(node): - """ - Reverse node input and output. - - Notes: - - Hooks are not reversed. - - Some nodes should be stopped or restarted before reversing. - - Nodes with in-/output buffers should be stopped before reversing. - """ - return vn.node_reverse(node) - - -def node_start(node): - """ - Start a node. - - Notes: - - Nodes are not meant to be started again without stopping first. - """ - return vn.node_start(node) - - -def node_stop(node): - """ - Stop a node. - - Notes: - - Use before starting a node again. - - May delete in-/output buffers of a node. - """ - return vn.node_stop(node) - - -def node_to_json(node): - """ - Return the node configuration as json object. - - Notes: - - Node configuration may not match self made configurations. - - Node configuration does not contain node name. - """ - json_str = vn.node_to_json_str(node) - json_obj = json.loads(json_str) - return json_obj - - -def node_to_json_str(node): - """ - Returns the node configuration as string. - - Notes: - - Node configuration may not match self made configurations. - - Node configuration does not contain node name. - """ - return vn.node_to_json_str(node) - - -@_warn_if_not_implemented -def node_write(node, samples, count): - """ - Write samples from a SamplesArray, fully or as block slice, into a node. - - Args: - node: Node handle. - samples: Either a SamplesArray or a tuple (SamplesArray, index/slice). - count: Number of samples to write. - - Returns: - int: Number of samples written on success, or -1 if not implemented. - - Notes: - - Return value may vary depending on node type. - """ - if isinstance(samples, SamplesArray): - return vn.node_write(node, samples._get_block_handle(), count) - elif isinstance(samples, tuple): - smpls = samples[0] - if not isinstance(samples[1], slice): - raise ValueError("Invalid samples Parameter") - start, stop, _ = samples[1].indices(len(smpls)) - - # check for length mismatch - if (stop - start) != count: - raise ValueError("Slice length and sample count do not match.") - # check for out of bounds - if stop > len(smpls): - raise IndexError("Out of bounds") - - # get block handle - handle = samples[0]._get_block_handle(start) - - return vn.node_write(node, handle, count) - else: - logger.warning("Invalid samples Parameter") - + Args: + idx (int): Node index to store packed sample in. + ts_orig (Optional[int]): Supposed creation time in ns. + ts_recv (Optional[int]): Supposed arrival time in ns. + values (Union[float, list[float], sample]): + - Packed sample will only hold referenced values. + seq (int): supposed sequence number of the sample. + """ + if seq < 0: + raise ValueError("seq has to be positive") + + Node._ensure_capacity(self._smps, idx + 1) + if len(self._smps) <= idx: + self._smps.grow(idx + 1 - len(self._smps)) + + if isinstance(values, (float, int)): + self._smps[idx] = vn.sample_pack( + [values], + ts_orig, + ts_recv, + seq, + ) + elif isinstance(values, list): + self._smps[idx] = vn.sample_alloc(len(values)) + self._smps[idx] = vn.sample_pack(values, ts_orig, ts_recv, seq) + else: # assume a PyCapsule + self._smps[idx] = vn.sample_pack(values, ts_orig, ts_recv) + + def unpack_to( + self, + r_idx: int, + target_node, + w_idx: int, + ts_orig: Optional[int] = None, + ts_recv: Optional[int] = None, + ): + """ + Unpacks a given sample to a destined target. -def sample_length(*args): - vn.sample_length(*args) + Args: + r_idx (int): Originating Node index to read from. + ts_orig (Optional[int]): Supposed creation time in ns. + ts_recv (Optional[int]): Supposed arrival time in ns. + target_node (Node): Target node. + w_idx (int): Target Node index to unpack to. + """ + Node._ensure_capacity(self._smps, r_idx + 1) + Node._ensure_capacity(target_node._smps, w_idx + 1) + vn.sample_unpack( + self._smps.get_block(r_idx), + target_node._smps.get_block(w_idx), + ts_orig, + ts_recv, + ) -def sample_pack(*args): - vn.sample_pack(*args) + def sample_details(self, idx): + """ + Retrieve a dict with information about a sample. + Keys: + `sequence` (int): Sequence number of the sample. + `length` (int): Sample length. + `capacity` (int): Allocated sample length. + `flags` (int): Number representing flags set of the sample. + `refcnt` (int): Reference count of the given sample. + `ts_origin` (float): Supposed timestamp of creation. + `ts_received` (float): Supposed timestamp of arrival. -def sample_unpack(*args): - vn.sample_unpack(*args) + Returns: + Dict with listed keys and values. + """ + return vn.sample_details(self._smps[idx]) From d47ef143131f42729bbca688a46c4381a79b3d35 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 15 Dec 2025 20:33:02 +0100 Subject: [PATCH 28/32] Update stubs to match changes Signed-off-by: Kevin Vu te Laar --- python/villas/node/binding.pyi | 125 ++++++++++++++++++-------- python/villas/node/python_binding.pyi | 29 +++--- 2 files changed, 100 insertions(+), 54 deletions(-) diff --git a/python/villas/node/binding.pyi b/python/villas/node/binding.pyi index 995819a6a..09e5bdb8c 100644 --- a/python/villas/node/binding.pyi +++ b/python/villas/node/binding.pyi @@ -8,48 +8,95 @@ SPDX-License-Identifier: Apache-2.0 """ from _typeshed import Incomplete -from typing import Any, Callable +from typing import Any, Callable, Optional +Capsule = Any logger: Incomplete -class SamplesArray: - def __init__(self, length: int) -> None: ... - def __len__(self) -> int: ... - def __getitem__(self, idx: int | slice): ... - def __copy__(self) -> None: ... - def __deepcopy__(self) -> None: ... - def _warn_if_not_implemented( func: Callable[..., Any], ) -> Callable[..., Any]: ... -def memory_init(*args): ... -def node_check(node): ... -def node_destroy(node): ... -def node_details(node): ... -def node_input_signals_max_cnt(node): ... -def node_is_enabled(node): ... -def node_is_valid_name(name: str): ... -def node_name(node): ... -def node_name_full(node): ... -def node_name_short(node): ... -def node_netem_fds(*args): ... -def node_new(config, uuid: str | None = None): ... -def node_output_signals_max_cnt(node): ... -def node_pause(node): ... -def node_poll_fds(*args): ... -def node_prepare(node): ... -@_warn_if_not_implemented -def node_read(node, samples, sample_length, count): ... -def node_restart(node): ... -def node_resume(node): ... -@_warn_if_not_implemented -def node_reverse(node): ... -def node_start(node): ... -def node_stop(node): ... -def node_to_json(node): ... -def node_to_json_str(node): ... -@_warn_if_not_implemented -def node_write(node, samples, count): ... -def sample_length(*args) -> None: ... -def sample_pack(*args) -> None: ... -def sample_unpack(*args) -> None: ... + +class Node: + class _SampleSlice: + node: Incomplete + idx: Incomplete + + def __init__(self, node, idx) -> None: ... + def details(self): ... + def read_from(self, sample_length, count: Incomplete | None = None): ... + def write_to(self, node, count: Incomplete | None = None): ... + def pack_from( + self, + values: float | list[float] | Capsule, + ts_origin: int | None, + ts_received: int | None, + seq: int = 0, + ): ... + def unpack_to( + self, + target: Capsule, + ts_origin: int | None, + ts_received: int | None, + ): ... + + config: Incomplete + + def __init__(self, config, uuid: str | None = None, size: int = 0) -> None: ... + def __del__(self) -> None: ... + def __getitem__(self, idx: int | slice): ... + def __setitem__(self, obj) -> None: ... + def __len__(self) -> int: ... + def __copy__(self) -> None: ... + def __deepcopy__(self) -> None: ... + @staticmethod + def memory_init(hugepages: int): ... + def check(self): ... + def details(self): ... + def input_signals_max_cnt(self): ... + def is_enabled(self): ... + @staticmethod + def is_valid_name(name: str): ... + def name(self): ... + def name_full(self): ... + def name_short(self): ... + def output_signals_max_cnt(self): ... + def pause(self): ... + def prepare(self): ... + def read_from( + self, + sample_length, + cnt: int | None = None, + idx: Incomplete | None = None, + ): ... + def restart(self): ... + def resume(self): ... + def reverse(self): ... + def start(self): ... + def stop(self): ... + def to_json(self): ... + def to_json_str(self): ... + def write_to( + self, + node, + cnt: int | None = None, + idx: Incomplete | None = None, + ): ... + def sample_length(self, idx: int): ... + def pack_from( + self, + idx: int, + ts_orig: int | None, + ts_recv: int | None, + values: float | list[float] | Capsule, + seq: int = 0, + ): ... + def unpack_to( + self, + r_idx: int, + target_node, + w_idx: int, + ts_orig: int | None = None, + ts_recv: int | None = None, + ): ... + def sample_details(self, idx): ... diff --git a/python/villas/node/python_binding.pyi b/python/villas/node/python_binding.pyi index 81b91d128..bce640600 100644 --- a/python/villas/node/python_binding.pyi +++ b/python/villas/node/python_binding.pyi @@ -8,16 +8,16 @@ SPDX-License-Identifier: Apache-2.0 """ import typing -from typing import Any, overload +from typing import Any, Optional, overload -Array = Any capsule = Any -timespec = Any class SamplesArray: def __init__(self, len: int) -> None: ... def bulk_alloc(self, arg0: int, arg1: int, arg2: int) -> None: ... + def clear(self) -> int: ... def get_block(self, arg0: int) -> capsule: ... + def grow(self, arg0: int) -> int: ... def __getitem__(self, arg0: int) -> capsule: ... def __iter__(self) -> typing.Iterator[capsule]: ... def __setitem__(self, arg0: int, arg1: capsule) -> None: ... @@ -32,14 +32,12 @@ def node_is_valid_name(arg0: str) -> bool: ... def node_name(arg0: capsule) -> str: ... def node_name_full(arg0: capsule) -> str: ... def node_name_short(arg0: capsule) -> str: ... -def node_netem_fds(arg0: capsule, arg1: int) -> int: ... def node_new(arg0: str, arg1: str) -> capsule: ... def node_output_signals_max_cnt(arg0: capsule) -> int: ... def node_pause(arg0: capsule) -> int: ... -def node_poll_fds(arg0: capsule, arg1: int) -> int: ... def node_prepare(arg0: capsule) -> int: ... @overload -def node_read(arg0: capsule, arg1: Array, arg2: int) -> int: ... +def node_read(arg0: capsule, arg1: SamplesArray, arg2: int) -> int: ... @overload def node_read(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] def node_restart(arg0: capsule) -> int: ... @@ -49,22 +47,23 @@ def node_start(arg0: capsule) -> int: ... def node_stop(arg0: capsule) -> int: ... def node_to_json_str(arg0: capsule) -> str: ... @overload -def node_write(arg0: capsule, arg1: Array, arg2: int) -> int: ... +def node_write(arg0: capsule, arg1: SamplesArray, arg2: int) -> int: ... @overload def node_write(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] def sample_alloc(arg0: int) -> capsule: ... def sample_decref(arg0: capsule) -> None: ... def sample_length(arg0: capsule) -> int: ... +@overload +def sample_pack(arg0: capsule, arg1: Optional[int], arg2: Optional[int]) -> None: ... +@overload def sample_pack( - arg0: int, arg1: timespec, arg2: timespec, arg3: int, arg4: float + arg0: list, arg1: Optional[int], arg2: Optional[int], arg3: int ) -> capsule: ... def sample_unpack( arg0: capsule, - arg1: int, - arg2: timespec, - arg3: timespec, - arg4: int, - arg5: int, - arg6: float, + arg1: capsule, + arg2: Optional[int], + arg3: Optional[int], ) -> None: ... -def smps_array(arg0: int) -> Array: ... +def sample_details(arg0: capsule) -> dict: ... +def smps_array(arg0: int) -> SamplesArray: ... From d075db1e864d2e390c1cef58805572801ac93bbc Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Mon, 15 Dec 2025 20:37:11 +0100 Subject: [PATCH 29/32] Update tests to match binding changes Updated function names and evoking syntax. Added test for: - Automatic SamplesArray sizing - `sample_pack()`, `sample_unpack()` accessible via `pack_from()`, `unpack_to` - `samples_detail()` Signed-off-by: Kevin Vu te Laar --- .../python/test_binding_wrapper.py | 167 ++++++++++-------- tests/unit/python/test_binding_wrapper.py | 75 ++++---- 2 files changed, 124 insertions(+), 118 deletions(-) diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py index fbb2759ff..4c1de7e7e 100644 --- a/tests/integration/python/test_binding_wrapper.py +++ b/tests/integration/python/test_binding_wrapper.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import villas.node.binding as b +from villas.node.binding import Node class BindingWrapperIntegrationTests(unittest.TestCase): @@ -16,61 +16,54 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = b.node_new(self.config, self.node_uuid) + self.test_node = Node(self.config, self.node_uuid) except Exception as e: self.fail(f"new_node err: {e}") - def tearDown(self): - try: - b.node_stop(self.test_node) - b.node_destroy(self.test_node) - except Exception as e: - self.fail(f"node cleanup error: {e}") - def test_activity_changes(self): try: - b.node_check(self.test_node) - b.node_prepare(self.test_node) + self.test_node.check() + self.test_node.prepare() # starting twice - self.assertEqual(0, b.node_start(self.test_node)) + self.assertEqual(0, self.test_node.start()) # check if the node is running - self.assertTrue(b.node_is_enabled(self.test_node)) + self.assertTrue(self.test_node.is_enabled()) # pausing twice - self.assertEqual(0, b.node_pause(self.test_node)) - self.assertEqual(-1, b.node_pause(self.test_node)) + self.assertEqual(0, self.test_node.pause()) + self.assertEqual(-1, self.test_node.pause()) # resuming - self.assertEqual(0, b.node_resume(self.test_node)) + self.assertEqual(0, self.test_node.resume()) # stopping twice - self.assertEqual(0, b.node_stop(self.test_node)) - self.assertEqual(0, b.node_stop(self.test_node)) + self.assertEqual(0, self.test_node.stop()) + self.assertEqual(0, self.test_node.stop()) # restarting - b.node_restart(self.test_node) + self.test_node.restart() # check if everything still works after restarting - b.node_pause(self.test_node) - b.node_resume(self.test_node) - b.node_stop(self.test_node) - b.node_start(self.test_node) + self.test_node.pause() + self.test_node.resume() + self.test_node.stop() + self.test_node.start() except Exception as e: self.fail(f" err: {e}") def test_reverse_node(self): try: - self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, self.test_node.input_signals_max_cnt()) + self.assertEqual(0, self.test_node.output_signals_max_cnt()) - self.assertEqual(0, b.node_reverse(self.test_node)) + self.assertEqual(0, self.test_node.reverse()) # input and output hooks/details are not reversed # input and output are reversed, can be seen with wireshark and # function test_rw_socket_and_reverse() below - self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) - self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(1, self.test_node.input_signals_max_cnt()) + self.assertEqual(0, self.test_node.output_signals_max_cnt()) except Exception as e: self.fail(f"Reversing node in and output failed: {e}") @@ -80,23 +73,23 @@ def test_reverse_node(self): # uuid can not match def test_config_from_string(self): try: - config_str = b.node_to_json_str(self.test_node) + config_str = self.test_node.to_json_str() config_obj = json.loads(config_str) config_copy_str = json.dumps(config_obj, indent=2) - test_node = b.node_new(config_copy_str) + test_node = Node(config_copy_str) self.assertEqual( re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - b.node_name_full(test_node), + test_node.name_full(), ), re.sub( r"^[^:]+: uuid=[0-9a-fA-F-]+, ", "", - b.node_name_full(self.test_node), + self.test_node.name_full(), ), ) except Exception as e: @@ -113,55 +106,43 @@ def test_rw_socket_and_reverse(self): config = json.dumps(obj, indent=2) id = str(uuid.uuid4()) - test_nodes[name] = b.node_new(config, id) + test_nodes[name] = Node(config, id) for node in test_nodes.values(): - if b.node_check(node): + if node.check(): raise RuntimeError("Failed to verify node configuration") - if b.node_prepare(node): - raise RuntimeError( - f"Failed to verify {b.node_name(node)} node config" - ) - b.node_start(node) - - # Arrays to store samples - send_smpls = b.SamplesArray(1) - intmdt_smpls = b.SamplesArray(100) - recv_smpls = b.SamplesArray(100) + if node.prepare(): + raise RuntimeError(f"Failed to verify {node.name()} node config") + node.start() for i in range(100): # Generate signals and send over send_socket + self.assertEqual(test_nodes["signal_generator"][i].read_from(2, 1), 1) self.assertEqual( - b.node_read(test_nodes["signal_generator"], send_smpls, 2, 1), + test_nodes["send_socket"][i].write_to( + test_nodes["signal_generator"], 1 + ), 1, ) - self.assertEqual( - b.node_write(test_nodes["send_socket"], send_smpls, 1), 1 - ) + self.assertEqual(test_nodes["signal_generator"].sample_length(0), 2) # read received signals and send them to recv_socket + self.assertEqual(test_nodes["intmdt_socket"].read_from(2, 100), 100) self.assertEqual( - b.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 2, 100), - 100, - ) - self.assertEqual( - b.node_write(test_nodes["intmdt_socket"], intmdt_smpls[0:50], 50), - 50, + test_nodes["intmdt_socket"][:30].write_to( + test_nodes["intmdt_socket"], 30 + ), + 30, ) self.assertEqual( - b.node_write(test_nodes["intmdt_socket"], intmdt_smpls[50:100], 50), - 50, + test_nodes["intmdt_socket"][30:].write_to(test_nodes["intmdt_socket"]), + 70, ) + # print(len(test_nodes["intmdt_socket"]._smps)) # confirm rev_socket signals - self.assertEqual( - b.node_read(test_nodes["recv_socket"], recv_smpls[0:50], 2, 50), - 50, - ) - self.assertEqual( - b.node_read(test_nodes["recv_socket"], recv_smpls[50:100], 2, 50), - 50, - ) + self.assertEqual(test_nodes["recv_socket"].read_from(2, 30), 30) + self.assertEqual(test_nodes["recv_socket"][30].read_from(2, 70), 70) # reversing in and outputs # stopping the socket is necessary to clean up buffers @@ -169,31 +150,65 @@ def test_rw_socket_and_reverse(self): # this can be confirmed when observing network traffic # node details do not represent this properly as of now for node in test_nodes.values(): - b.node_reverse(node) - b.node_stop(node) + node.reverse() + node.stop() for node in test_nodes.values(): - b.node_start(node) + node.start() - # if another 50 samples have not been allocated, - # sending 100 at once is impossible with recv_smpls + # if another 30+70 samples are not allocated, + # sending 100 at once is impossible self.assertEqual( - b.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + test_nodes["recv_socket"].write_to(test_nodes["recv_socket"], 100), + 100, ) # try writing as full slice self.assertEqual( - b.node_write(test_nodes["intmdt_socket"], recv_smpls[0:100], 100), + test_nodes["intmdt_socket"][0:100].write_to( + test_nodes["recv_socket"], 100 + ), 100, ) - # cleanup - for node in test_nodes.values(): - b.node_stop(node) - b.node_destroy(node) - except Exception as e: self.fail(f" err: {e}") + def test_sample_pack_unpack(self): + try: + self.test_node.pack_from(0, [0.01, 1.01, 2.01, 3.01, 4.01]) + self.test_node[1].pack_from( + [1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100 + ) + self.test_node[2].pack_from(42, int(1e9), int(1e9) + 100) + self.test_node[3].pack_from(self.test_node[1], int(1e9), int(1e9) + 100) + self.test_node[2].unpack_to(self.test_node[1], int(1e9), int(1e9) + 100) + self.assertEqual([42.0], self.test_node[1].details()["data"]) + self.test_node[0].unpack_to(self.test_node[1], int(2e9), int(2e9) + 100) + self.assertEqual( + [0.01, 1.01, 2.01, 3.01, 4.01], + self.test_node[1].details()["data"], + ) + self.test_node[0].unpack_to(self.test_node[2], int(2e9), int(2e9) + 100) + self.test_node[0].unpack_to(self.test_node[4], int(2e9), int(2e9) + 100) + self.test_node[1].unpack_to(self.test_node[2], int(2e9), int(2e9) + 100) + except Exception as e: + self.fail(f"err: {e}") + + def test_samplesarray_size(self): + try: + node_config = json.dumps(test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + node = Node(node_config, node_uuid, 100) + self.assertEqual(len(node), 100) + node[199].pack_from( + [1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100 + ) + self.assertEqual(len(node), 200) + node[199].unpack_to(node[299], int(2e9), int(2e9) + 100) + self.assertEqual(len(node), 300) + except Exception as e: + self.fail(f"err: {e}") + test_node_config = { "test_node": { diff --git a/tests/unit/python/test_binding_wrapper.py b/tests/unit/python/test_binding_wrapper.py index 781202078..b88b3eeab 100644 --- a/tests/unit/python/test_binding_wrapper.py +++ b/tests/unit/python/test_binding_wrapper.py @@ -8,7 +8,7 @@ import re import unittest import uuid -import villas.node.binding as b +from villas.node.binding import Node class BindingWrapperUnitTests(unittest.TestCase): @@ -16,26 +16,17 @@ def setUp(self): try: self.config = json.dumps(test_node_config, indent=2) self.node_uuid = str(uuid.uuid4()) - self.test_node = b.node_new(self.config, self.node_uuid) + self.test_node = Node(self.config, self.node_uuid) config = json.dumps(signal_test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - self.signal_test_node = b.node_new(config, node_uuid) + self.signal_test_node = Node(config, node_uuid) except Exception as e: self.fail(f"new_node err: {e}") - def tearDown(self): - try: - b.node_stop(self.test_node) - b.node_destroy(self.test_node) - b.node_stop(self.signal_test_node) - b.node_destroy(self.signal_test_node) - except Exception as e: - self.fail(f"node cleanup error: {e}") - def test_start(self): try: - self.assertEqual(0, b.node_start(self.test_node)) - self.assertEqual(0, b.node_start(self.signal_test_node)) + self.assertEqual(0, self.test_node.start()) + self.assertEqual(0, self.signal_test_node.start()) except Exception as e: self.fail(f"err: {e}") @@ -46,9 +37,9 @@ def test_start(self): ) def test_start_err(self): try: - self.assertEqual(0, b.node_start(self.test_node)) + self.assertEqual(0, self.test_node.start()) with self.assertRaises((AssertionError, RuntimeError)): - b.node_start(self.test_node) + self.test_node.start() except Exception as e: self.fail(f"err: {e}") @@ -56,54 +47,54 @@ def test_new(self): try: node_config = json.dumps(test_node_config, indent=2) node_uuid = str(uuid.uuid4()) - node = b.node_new(node_config, node_uuid) + node = Node(node_config, node_uuid) self.assertIsNotNone(node) except Exception as e: self.fail(f"err: {e}") def test_check(self): try: - b.node_check(self.test_node) + self.test_node.check() except Exception as e: self.fail(f"err: {e}") def test_prepare(self): try: - b.node_prepare(self.test_node) + self.test_node.prepare() except Exception as e: self.fail(f"err: {e}") def test_is_enabled(self): try: - self.assertTrue(b.node_is_enabled(self.test_node)) + self.assertTrue(self.test_node.is_enabled()) except Exception as e: self.fail(f"err: {e}") def test_pause(self): try: - self.assertEqual(-1, b.node_pause(self.test_node)) - self.assertEqual(-1, b.node_pause(self.test_node)) + self.assertEqual(-1, self.test_node.pause()) + self.assertEqual(-1, self.test_node.pause()) except Exception as e: self.fail(f"err: {e}") def test_resume(self): try: - self.assertEqual(0, b.node_resume(self.test_node)) + self.assertEqual(0, self.test_node.resume()) except Exception as e: self.fail(f"err: {e}") def test_stop(self): try: - self.assertEqual(0, b.node_start(self.test_node)) - self.assertEqual(0, b.node_stop(self.test_node)) - self.assertEqual(0, b.node_stop(self.test_node)) + self.assertEqual(0, self.test_node.start()) + self.assertEqual(0, self.test_node.stop()) + self.assertEqual(0, self.test_node.stop()) except Exception as e: self.fail(f"err: {e}") def test_restart(self): try: - self.assertEqual(0, b.node_restart(self.test_node)) - self.assertEqual(0, b.node_restart(self.test_node)) + self.assertEqual(0, self.test_node.restart()) + self.assertEqual(0, self.test_node.restart()) except Exception as e: self.fail(f"err: {e}") @@ -112,14 +103,14 @@ def test_node_name(self): # remove color codes before checking for equality self.assertEqual( "test_node(socket)", - re.sub(r"\x1b\[[0-9;]*m", "", b.node_name(self.test_node)), + re.sub(r"\x1b\[[0-9;]*m", "", self.test_node.name()), ) except Exception as e: self.fail(f"err: {e}") def test_node_name_short(self): try: - self.assertEqual("test_node", b.node_name_short(self.test_node)) + self.assertEqual("test_node", self.test_node.name_short()) except Exception as e: self.fail(f"err: {e}") @@ -133,7 +124,7 @@ def test_node_name_full(self): + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", - re.sub(r"\x1b\[[0-9;]*m", "", b.node_name_full(node)), + re.sub(r"\x1b\[[0-9;]*m", "", node.name_full()), ) except Exception as e: self.fail(f"err: {e}") @@ -144,33 +135,33 @@ def test_details(self): "layer=udp, " + "in.address=0.0.0.0:12000, " + "out.address=127.0.0.1:12001", - b.node_details(self.test_node), + self.test_node.details(), ) except Exception as e: self.fail(f"err: {e}") def test_node_to_json(self): try: - if not isinstance(b.node_to_json(self.test_node), dict): + if not isinstance(self.test_node.to_json(), dict): self.fail("Not a JSON object (dict)") except Exception as e: self.fail(f"err: {e}") def test_node_to_json_str(self): try: - json.loads(b.node_to_json_str(self.test_node)) + json.loads(self.test_node.to_json_str()) except Exception as e: self.fail(f"err: {e}") def test_input_signals_max_cnt(self): try: - self.assertEqual(1, b.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(1, self.test_node.input_signals_max_cnt()) except Exception as e: self.fail(f"err: {e}") def test_node_output_signals_max_cnt(self): try: - self.assertEqual(0, b.node_output_signals_max_cnt(self.test_node)) + self.assertEqual(0, self.test_node.output_signals_max_cnt()) except Exception as e: self.fail(f"err: {e}") @@ -185,21 +176,21 @@ def test_node_is_valid_name(self): valid_names = ["32_characters_long_strings_valid", "valid_name"] for name in invalid_names: - self.assertFalse(b.node_is_valid_name(name)) + self.assertFalse(Node.is_valid_name(name)) for name in valid_names: - self.assertTrue(b.node_is_valid_name(name)) + self.assertTrue(Node.is_valid_name(name)) except Exception as e: self.fail(f"err: {e}") def test_reverse(self): try: # socket has reverse() implemented, expected return 0 - self.assertEqual(0, b.node_reverse(self.test_node)) - self.assertEqual(0, b.node_reverse(self.test_node)) + self.assertEqual(0, self.test_node.reverse()) + self.assertEqual(0, self.test_node.reverse()) # signal.v2 has not reverse() implemented, expected return 1 - self.assertEqual(-1, b.node_reverse(self.signal_test_node)) - self.assertEqual(-1, b.node_reverse(self.signal_test_node)) + self.assertEqual(-1, self.signal_test_node.reverse()) + self.assertEqual(-1, self.signal_test_node.reverse()) except Exception as e: self.fail(f"err: {e}") From f0c4eb81470ae67776980039ae1ca2a8e26c9b5e Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 16 Dec 2025 09:29:14 +0100 Subject: [PATCH 30/32] Enforce pre-commit formatting Reformat python files to adhere to the updated 90-character line-length setting in pyproject.toml. Signed-off-by: Kevin Vu te Laar --- python/villas/node/binding.py | 4 +--- tests/integration/python/test_binding_wrapper.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py index d9b47b282..b56101ab2 100644 --- a/python/villas/node/binding.py +++ b/python/villas/node/binding.py @@ -83,9 +83,7 @@ def pack_from( seq, ) else: - return self.node.pack_from( - self.idx, values, ts_origin, ts_received, seq - ) + return self.node.pack_from(self.idx, values, ts_origin, ts_received, seq) def unpack_to( self, diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py index 4c1de7e7e..74e3cb250 100644 --- a/tests/integration/python/test_binding_wrapper.py +++ b/tests/integration/python/test_binding_wrapper.py @@ -200,9 +200,7 @@ def test_samplesarray_size(self): node_uuid = str(uuid.uuid4()) node = Node(node_config, node_uuid, 100) self.assertEqual(len(node), 100) - node[199].pack_from( - [1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100 - ) + node[199].pack_from([1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100) self.assertEqual(len(node), 200) node[199].unpack_to(node[299], int(2e9), int(2e9) + 100) self.assertEqual(len(node), 300) From 00b652801aad8f67fc911b0c4e427e29e59098e9 Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Tue, 16 Dec 2025 13:00:34 +0100 Subject: [PATCH 31/32] Replace C with C++ casting, sample pack/unpack fix Replace C-style casts with C++-style casts. `sample_unpack()` always unpacks source sample timespec instead of default or user-specified values. `sample_pack()` uses user-specified if provided; otherwise falls back to source sample timespec when packing from a sample, or to the system clock when packing from a list. Modified stubs to reflect changes. Signed-off-by: Kevin Vu te Laar --- python/binding/capi_python_binding.cpp | 160 +++++++++++------- python/villas/node/binding.py | 12 +- python/villas/node/binding.pyi | 4 - python/villas/node/python_binding.pyi | 2 - .../python/test_binding_wrapper.py | 12 +- 5 files changed, 102 insertions(+), 88 deletions(-) diff --git a/python/binding/capi_python_binding.cpp b/python/binding/capi_python_binding.cpp index 680e0f8af..66290a0a0 100644 --- a/python/binding/capi_python_binding.cpp +++ b/python/binding/capi_python_binding.cpp @@ -47,7 +47,9 @@ class SamplesArray { delete[] smps; } - void *get_block(unsigned int start) { return (void *)&smps[start]; } + void *get_block(unsigned int start) { + return reinterpret_cast(&smps[start]); + } void bulk_alloc(unsigned int start_idx, unsigned int stop_idx, unsigned int smpl_len) { @@ -125,39 +127,51 @@ struct timespec ns_to_timespec(int64_t time_ns) { PYBIND11_MODULE(python_binding, m) { m.def("memory_init", &memory_init); - m.def("node_check", [](void *n) -> int { return node_check((vnode *)n); }); + m.def("node_check", [](void *n) -> int { + return node_check(reinterpret_cast(n)); + }); - m.def("node_destroy", - [](void *n) -> int { return node_destroy((vnode *)n); }); + m.def("node_destroy", [](void *n) -> int { + return node_destroy(reinterpret_cast(n)); + }); m.def( "node_details", - [](void *n) -> const char * { return node_details((vnode *)n); }, + [](void *n) -> const char * { + return node_details(reinterpret_cast(n)); + }, py::return_value_policy::copy); m.def("node_input_signals_max_cnt", [](void *n) -> unsigned { - return node_input_signals_max_cnt((vnode *)n); + return node_input_signals_max_cnt(reinterpret_cast(n)); }); - m.def("node_is_enabled", - [](void *n) -> bool { return node_is_enabled((const vnode *)n); }); + m.def("node_is_enabled", [](void *n) -> bool { + return node_is_enabled(reinterpret_cast(n)); + }); m.def("node_is_valid_name", [](const char *name) -> bool { return node_is_valid_name(name); }); m.def( "node_name", - [](void *n) -> const char * { return node_name((vnode *)n); }, + [](void *n) -> const char * { + return node_name(reinterpret_cast(n)); + }, py::return_value_policy::copy); m.def( "node_name_full", - [](void *n) -> const char * { return node_name_full((vnode *)n); }, + [](void *n) -> const char * { + return node_name_full(reinterpret_cast(n)); + }, py::return_value_policy::copy); m.def( "node_name_short", - [](void *n) -> const char * { return node_name_short((vnode *)n); }, + [](void *n) -> const char * { + return node_name_short(reinterpret_cast(n)); + }, py::return_value_policy::copy); m.def( @@ -173,8 +187,8 @@ PYBIND11_MODULE(python_binding, m) { json_t *inner = json_object_iter_value(it); if (json_is_object(inner)) { // create node with name - return (vnode *)villas::node::NodeFactory::make( - json_object_iter_value(it), id, json_object_iter_key(it)); + return reinterpret_cast(villas::node::NodeFactory::make( + json_object_iter_value(it), id, json_object_iter_key(it))); } else { // create node without name char *capi_str = json_dumps(json, 0); auto ret = node_new(id_str, capi_str); @@ -186,33 +200,44 @@ PYBIND11_MODULE(python_binding, m) { py::return_value_policy::take_ownership); m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { - return node_output_signals_max_cnt((vnode *)n); + return node_output_signals_max_cnt(reinterpret_cast(n)); }); - m.def("node_pause", [](void *n) -> int { return node_pause((vnode *)n); }); + m.def("node_pause", [](void *n) -> int { + return node_pause(reinterpret_cast(n)); + }); - m.def("node_prepare", - [](void *n) -> int { return node_prepare((vsample *)n); }); + m.def("node_prepare", [](void *n) -> int { + return node_prepare(reinterpret_cast(n)); + }); m.def("node_read", [](void *n, SamplesArray &a, unsigned cnt) -> int { - return node_read((vnode *)n, a.get_smps(), cnt); + return node_read(reinterpret_cast(n), a.get_smps(), cnt); }); m.def("node_read", [](void *n, void *smpls, unsigned cnt) -> int { - return node_read((vnode *)n, (vsample **)smpls, cnt); + return node_read(reinterpret_cast(n), + reinterpret_cast(smpls), cnt); }); - m.def("node_restart", - [](void *n) -> int { return node_restart((vnode *)n); }); + m.def("node_restart", [](void *n) -> int { + return node_restart(reinterpret_cast(n)); + }); - m.def("node_resume", [](void *n) -> int { return node_resume((vnode *)n); }); + m.def("node_resume", [](void *n) -> int { + return node_resume(reinterpret_cast(n)); + }); - m.def("node_reverse", - [](void *n) -> int { return node_reverse((vnode *)n); }); + m.def("node_reverse", [](void *n) -> int { + return node_reverse(reinterpret_cast(n)); + }); - m.def("node_start", [](void *n) -> int { return node_start((vnode *)n); }); + m.def("node_start", [](void *n) -> int { + return node_start(reinterpret_cast(n)); + }); - m.def("node_stop", [](void *n) -> int { return node_stop((vnode *)n); }); + m.def("node_stop", + [](void *n) -> int { return node_stop(reinterpret_cast(n)); }); m.def("node_to_json_str", [](void *n) -> py::str { auto json = reinterpret_cast(n)->toJson(); @@ -226,11 +251,12 @@ PYBIND11_MODULE(python_binding, m) { }); m.def("node_write", [](void *n, SamplesArray &a, unsigned cnt) -> int { - return node_write((vnode *)n, a.get_smps(), cnt); + return node_write(reinterpret_cast(n), a.get_smps(), cnt); }); m.def("node_write", [](void *n, void *smpls, unsigned cnt) -> int { - return node_write((vnode *)n, (vsample **)smpls, cnt); + return node_write(reinterpret_cast(n), + reinterpret_cast(smpls), cnt); }); m.def( @@ -243,13 +269,13 @@ PYBIND11_MODULE(python_binding, m) { // Decrease reference count and release memory if last reference was held. m.def("sample_decref", [](void *smps) -> void { - auto smp = (vsample **)smps; + auto smp = reinterpret_cast(smps); sample_decref(*smp); }); m.def("sample_length", [](void *smp) -> unsigned { if (smp) { - return sample_length((vsample *)smp); + return sample_length(reinterpret_cast(smp)); } else { return -1; } @@ -259,15 +285,15 @@ PYBIND11_MODULE(python_binding, m) { "sample_pack", [](void *s, std::optional ts_origin_ns, std::optional ts_received_ns) -> vsample * { - struct timespec ts_origin = - ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); - struct timespec ts_received = - ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); - - auto smp = (villas::node::Sample *)s; + auto smp = reinterpret_cast(s); uint64_t *seq = &smp->sequence; unsigned len = smp->length; - double *values = (double *)smp->data; + double *values = reinterpret_cast(smp->data); + + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : smp->ts.origin; + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : smp->ts.received; return sample_pack(seq, &ts_origin, &ts_received, len, values); }, @@ -283,52 +309,56 @@ PYBIND11_MODULE(python_binding, m) { ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); unsigned values_len = values.size(); - double cvalues[values.size()]; + double cvalues[values_len]; + for (unsigned int i = 0; i < values_len; ++i) { cvalues[i] = values[i].cast(); } uint64_t sequence = seq; - auto tmp = (void *)sample_pack(&sequence, &ts_origin, &ts_received, - values_len, cvalues); - return tmp; + return reinterpret_cast(sample_pack( + &sequence, &ts_origin, &ts_received, values_len, cvalues)); }, py::return_value_policy::reference); m.def( "sample_unpack", - [](void *ss, void *ds, std::optional ts_origin_ns, - std::optional ts_received_ns) -> void { - struct timespec ts_origin = - ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); - struct timespec ts_received = - ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); + [](void *ss, void *ds) -> void { + auto dSmp = reinterpret_cast(ds); + auto srcSmp = reinterpret_cast(ss); + auto &destSmp = *dSmp; - auto srcSmp = (villas::node::Sample **)ss; - auto destSmp = (villas::node::Sample **)ds; - - if (!*srcSmp) { + if (!srcSmp) { throw std::runtime_error("Tried to unpack empty sample!"); } - if (!*destSmp) { - *destSmp = (villas::node::Sample *)sample_alloc((*srcSmp)->length); - } else if ((*destSmp)->capacity < (*srcSmp)->length) { - sample_decref(*(vsample **)destSmp); - *destSmp = (villas::node::Sample *)sample_alloc((*srcSmp)->length); + if (!destSmp) { + goto alloc; + } + if (destSmp->capacity < srcSmp->length) { + sample_decref(reinterpret_cast(destSmp)); + goto alloc; + } + if (0) { + alloc: + *dSmp = reinterpret_cast( + sample_alloc(srcSmp->length)); + destSmp = *dSmp; } - uint64_t *seq = &(*destSmp)->sequence; - int *flags = &(*destSmp)->flags; - unsigned *len = &(*destSmp)->length; - double *values = (double *)(*destSmp)->data; + uint64_t *seq = &destSmp->sequence; + struct timespec *ts_origin = &destSmp->ts.origin; + struct timespec *ts_received = &destSmp->ts.received; + int *flags = &destSmp->flags; + unsigned *len = &destSmp->length; + double *values = reinterpret_cast(destSmp->data); - sample_unpack(*(vsample **)srcSmp, seq, &ts_origin, &ts_received, flags, - len, values); + sample_unpack(reinterpret_cast(srcSmp), seq, ts_origin, + ts_received, flags, len, values); }, py::return_value_policy::reference); m.def("sample_details", [](void *s) { - auto smp = (villas::node::Sample *)s; + auto smp = reinterpret_cast(s); if (!smp) { return py::dict(); } @@ -344,7 +374,7 @@ PYBIND11_MODULE(python_binding, m) { py::list data; for (unsigned int i = 0; i < smp->length; ++i) { - data.append((double)smp->data[i]); + data.append(static_cast(smp->data[i])); } d["data"] = data; @@ -364,7 +394,7 @@ PYBIND11_MODULE(python_binding, m) { if (a[idx]) { sample_decref(a[idx]); } - a[idx] = (vsample *)smp; + a[idx] = reinterpret_cast(smp); }) .def("__len__", &SamplesArray::size) .def("bulk_alloc", &SamplesArray::bulk_alloc) diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py index b56101ab2..05c0ce1f4 100644 --- a/python/villas/node/binding.py +++ b/python/villas/node/binding.py @@ -88,16 +88,12 @@ def pack_from( def unpack_to( self, target: Capsule, - ts_origin: Optional[int] = None, - ts_received: Optional[int] = None, ): if isinstance(target, self.__class__): return self.node.unpack_to( self.idx, target.node, target.idx, - ts_origin, - ts_received, ) else: raise ValueError( @@ -486,16 +482,12 @@ def unpack_to( r_idx: int, target_node, w_idx: int, - ts_orig: Optional[int] = None, - ts_recv: Optional[int] = None, ): """ Unpacks a given sample to a destined target. Args: r_idx (int): Originating Node index to read from. - ts_orig (Optional[int]): Supposed creation time in ns. - ts_recv (Optional[int]): Supposed arrival time in ns. target_node (Node): Target node. w_idx (int): Target Node index to unpack to. """ @@ -503,10 +495,8 @@ def unpack_to( Node._ensure_capacity(target_node._smps, w_idx + 1) vn.sample_unpack( - self._smps.get_block(r_idx), + self._smps[r_idx], target_node._smps.get_block(w_idx), - ts_orig, - ts_recv, ) def sample_details(self, idx): diff --git a/python/villas/node/binding.pyi b/python/villas/node/binding.pyi index 09e5bdb8c..99980bf6e 100644 --- a/python/villas/node/binding.pyi +++ b/python/villas/node/binding.pyi @@ -36,8 +36,6 @@ class Node: def unpack_to( self, target: Capsule, - ts_origin: int | None, - ts_received: int | None, ): ... config: Incomplete @@ -96,7 +94,5 @@ class Node: r_idx: int, target_node, w_idx: int, - ts_orig: int | None = None, - ts_recv: int | None = None, ): ... def sample_details(self, idx): ... diff --git a/python/villas/node/python_binding.pyi b/python/villas/node/python_binding.pyi index bce640600..469f1e8c9 100644 --- a/python/villas/node/python_binding.pyi +++ b/python/villas/node/python_binding.pyi @@ -62,8 +62,6 @@ def sample_pack( def sample_unpack( arg0: capsule, arg1: capsule, - arg2: Optional[int], - arg3: Optional[int], ) -> None: ... def sample_details(arg0: capsule) -> dict: ... def smps_array(arg0: int) -> SamplesArray: ... diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py index 74e3cb250..29604445a 100644 --- a/tests/integration/python/test_binding_wrapper.py +++ b/tests/integration/python/test_binding_wrapper.py @@ -181,16 +181,16 @@ def test_sample_pack_unpack(self): ) self.test_node[2].pack_from(42, int(1e9), int(1e9) + 100) self.test_node[3].pack_from(self.test_node[1], int(1e9), int(1e9) + 100) - self.test_node[2].unpack_to(self.test_node[1], int(1e9), int(1e9) + 100) + self.test_node[2].unpack_to(self.test_node[1]) self.assertEqual([42.0], self.test_node[1].details()["data"]) - self.test_node[0].unpack_to(self.test_node[1], int(2e9), int(2e9) + 100) + self.test_node[0].unpack_to(self.test_node[1]) self.assertEqual( [0.01, 1.01, 2.01, 3.01, 4.01], self.test_node[1].details()["data"], ) - self.test_node[0].unpack_to(self.test_node[2], int(2e9), int(2e9) + 100) - self.test_node[0].unpack_to(self.test_node[4], int(2e9), int(2e9) + 100) - self.test_node[1].unpack_to(self.test_node[2], int(2e9), int(2e9) + 100) + self.test_node[0].unpack_to(self.test_node[2]) + self.test_node[0].unpack_to(self.test_node[4]) + self.test_node[1].unpack_to(self.test_node[2]) except Exception as e: self.fail(f"err: {e}") @@ -202,7 +202,7 @@ def test_samplesarray_size(self): self.assertEqual(len(node), 100) node[199].pack_from([1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100) self.assertEqual(len(node), 200) - node[199].unpack_to(node[299], int(2e9), int(2e9) + 100) + node[199].unpack_to(node[299]) self.assertEqual(len(node), 300) except Exception as e: self.fail(f"err: {e}") From b999f8632d286d101e6498ee8dc6a5889f98c82d Mon Sep 17 00:00:00 2001 From: Kevin Vu te Laar Date: Thu, 18 Dec 2025 12:14:05 +0100 Subject: [PATCH 32/32] fix(ci): prevent mypy package root ambiguity Mypy was detecting the same files under multiple module names due to ambiguous package roots. Explicit package bases prevent ambiguity. Signed-off-by: Kevin Vu te Laar --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62a30c0c1..c422d6d5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -164,7 +164,7 @@ test:python: - pytest --verbose . - black --line-length=90 --extend-exclude=".*(\\.pyi|_pb2.py)$" --check . - flake8 --max-line-length=90 --extend-exclude="*.pyi,*_pb2.py" . - - mypy . + - mypy --explicit-package-bases . image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} needs: - job: "build:source: [fedora]"