diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b097746b..c422d6d5c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -164,7 +164,7 @@ test:python: - pytest --verbose . - black --line-length=90 --extend-exclude=".*(\\.pyi|_pb2.py)$" --check . - flake8 --max-line-length=90 --extend-exclude="*.pyi,*_pb2.py" . - - mypy . + - mypy --explicit-package-bases . image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} needs: - job: "build:source: [fedora]" @@ -182,6 +182,28 @@ test:cppcheck: - cppcheck.log expose_as: cppcheck +test:python_unit_integration: + stage: test + image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} + before_script: + # dependency from node.py, which gets imported because of __init__.py in node/villas/python/node + - pip install requests + script: + # build binding and symlink to villas.node folder + # helps with correct imports and compatiblity for CI, binding wrapper and binding install + - export PYTHONPATH=$PYTHONPATH:${PWD}/python + - cmake --build build ${CMAKE_BUILD_OPTS} --target python_binding + + - binding_path=$(find ${PWD}/build/python/binding/ -name "python_binding*.so" | head -n1) + - link_path=${PWD}/python/villas/node/$(basename $binding_path) + - ln -sf $binding_path $link_path + + - cmake --build build ${CMAKE_BUILD_OPTS} --target run-python-unit-tests run-python-integration-tests + - rm $link_path + needs: + - job: "build:source: [fedora]" + artifacts: true + test:unit: stage: test image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} diff --git a/CMakeLists.txt b/CMakeLists.txt index ff81c5d46..72038bb6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 -cmake_minimum_required(VERSION 3.14) +cmake_minimum_required(VERSION 3.15) project(villas-node VERSION 1.1.0 diff --git a/include/villas/node.h b/include/villas/node.h index 4ba8ed6f4..ba0e3d3be 100644 --- a/include/villas/node.h +++ b/include/villas/node.h @@ -6,6 +6,7 @@ */ #pragma once +#include #include #include @@ -67,11 +68,11 @@ unsigned sample_length(vsample *smp); void sample_decref(vsample *smp); -vsample *sample_pack(unsigned seq, struct timespec *ts_origin, +vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, unsigned len, double *values); -void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin, +void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, int *flags, unsigned *len, double *values); diff --git a/lib/node.cpp b/lib/node.cpp index b7453eb4b..52afc2320 100644 --- a/lib/node.cpp +++ b/lib/node.cpp @@ -440,7 +440,7 @@ Node *NodeFactory::make(json_t *json, const uuid_t &id, std::string type; Node *n; - if (json_is_object(json)) + if (!json_is_object(json)) throw ConfigError(json, "node-config-node", "Node configuration must be an object"); diff --git a/lib/node_capi.cpp b/lib/node_capi.cpp index ab96b1bdc..3a7ce676a 100644 --- a/lib/node_capi.cpp +++ b/lib/node_capi.cpp @@ -152,12 +152,12 @@ unsigned sample_length(vsample *s) { return smp->length; } -vsample *sample_pack(unsigned seq, struct timespec *ts_origin, +vsample *sample_pack(uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, unsigned len, double *values) { auto *smp = sample_alloc_mem(len); - smp->sequence = seq; + smp->sequence = *seq; smp->ts.origin = *ts_origin; smp->ts.received = *ts_received; smp->length = len; @@ -169,7 +169,7 @@ vsample *sample_pack(unsigned seq, struct timespec *ts_origin, return (vsample *)smp; } -void sample_unpack(vsample *s, unsigned *seq, struct timespec *ts_origin, +void sample_unpack(vsample *s, uint64_t *seq, struct timespec *ts_origin, struct timespec *ts_received, int *flags, unsigned *len, double *values) { auto *smp = (Sample *)s; diff --git a/lib/nodes/socket.cpp b/lib/nodes/socket.cpp index bcf49fd69..0912638ee 100644 --- a/lib/nodes/socket.cpp +++ b/lib/nodes/socket.cpp @@ -67,6 +67,9 @@ int villas::node::socket_init(NodeCompat *n) { auto *s = n->getData(); s->formatter = nullptr; + s->sd = -1; + s->in.buf = nullptr; + s->out.buf = nullptr; return 0; } @@ -77,6 +80,17 @@ int villas::node::socket_destroy(NodeCompat *n) { if (s->formatter) delete s->formatter; + if (s->in.buf != nullptr) { + memset(s->in.buf, 0, s->in.buflen); + delete[] s->in.buf; + s->in.buf = nullptr; + } + if (s->out.buf != nullptr) { + memset(s->out.buf, 0, s->out.buflen); + delete[] s->out.buf; + s->out.buf = nullptr; + } + return 0; } @@ -355,8 +369,16 @@ int villas::node::socket_stop(NodeCompat *n) { return ret; } - delete[] s->in.buf; - delete[] s->out.buf; + if (s->in.buf != nullptr) { + memset(s->in.buf, 0, s->in.buflen); + delete[] s->in.buf; + s->in.buf = nullptr; + } + if (s->out.buf != nullptr) { + memset(s->out.buf, 0, s->out.buflen); + delete[] s->out.buf; + s->out.buf = nullptr; + } return 0; } diff --git a/packaging/docker/Dockerfile.debian b/packaging/docker/Dockerfile.debian index 631e16f86..bd63e8eec 100644 --- a/packaging/docker/Dockerfile.debian +++ b/packaging/docker/Dockerfile.debian @@ -52,6 +52,8 @@ RUN apt-get update && \ libssl-dev \ libusb-1.0-0-dev \ libzmq3-dev \ + python3-dev \ + python3-pybind11 \ uuid-dev # Install unpackaged dependencies from source diff --git a/packaging/docker/Dockerfile.debian-multiarch b/packaging/docker/Dockerfile.debian-multiarch index 6007a9efd..a91de343f 100644 --- a/packaging/docker/Dockerfile.debian-multiarch +++ b/packaging/docker/Dockerfile.debian-multiarch @@ -58,6 +58,8 @@ RUN apt-get update && \ libssl-dev:${ARCH} \ libusb-1.0-0-dev:${ARCH} \ libzmq3-dev:${ARCH} \ + python3-dev:${ARCH} \ + python3-pybind11:${ARCH} \ uuid-dev:${ARCH} ADD cmake/toolchains/debian-${ARCH}.cmake / diff --git a/packaging/docker/Dockerfile.fedora b/packaging/docker/Dockerfile.fedora index 2beee634f..bc5549fc7 100644 --- a/packaging/docker/Dockerfile.fedora +++ b/packaging/docker/Dockerfile.fedora @@ -27,7 +27,7 @@ RUN dnf -y install \ openssh-clients \ jq nmap-ncat \ iproute iproute-tc \ - python python-devel python-pip \ + python-devel python-pip \ gdb gdb-gdbserver \ cppcheck \ xmlto dblatex rubygem-asciidoctor \ @@ -63,6 +63,7 @@ RUN dnf -y install \ openssl-devel \ protobuf-c-devel \ protobuf-devel \ + pybind11-devel \ spdlog-devel \ zeromq-devel diff --git a/packaging/docker/Dockerfile.rocky b/packaging/docker/Dockerfile.rocky index 384bf6d68..77da22c93 100644 --- a/packaging/docker/Dockerfile.rocky +++ b/packaging/docker/Dockerfile.rocky @@ -53,7 +53,9 @@ RUN dnf -y install \ nanomsg-devel \ libnice-devel \ libre-devel \ - libwebsockets-devel + libwebsockets-devel \ + python-devel \ + pybind11-devel # Install unpackaged dependencies from source ADD packaging/patches /deps/patches diff --git a/packaging/docker/Dockerfile.ubuntu b/packaging/docker/Dockerfile.ubuntu index 5a4a0c506..9e749a65e 100644 --- a/packaging/docker/Dockerfile.ubuntu +++ b/packaging/docker/Dockerfile.ubuntu @@ -59,6 +59,8 @@ RUN apt-get update && \ libusb-1.0-0-dev \ libwebsockets-dev \ libzmq3-dev \ + python3-dev \ + python3-pybind11 \ uuid-dev # Install unpackaged dependencies from source diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f9aa07f2f..e0f32dbc9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,6 +4,8 @@ # SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 +add_subdirectory(binding) + if(DEFINED PROTOBUF_COMPILER AND PROTOBUF_FOUND) add_custom_command( OUTPUT diff --git a/python/binding/CMakeLists.txt b/python/binding/CMakeLists.txt new file mode 100644 index 000000000..d1b346da1 --- /dev/null +++ b/python/binding/CMakeLists.txt @@ -0,0 +1,33 @@ +# Python-binding CMakeLists. +# +# Author: Kevin Vu te Laar +# SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-License-Identifier: Apache-2.0 + +set(PYBIND11_FINDPYTHON ON) +find_package(pybind11 CONFIG) + +if(pybind11_FOUND) + find_package(Python3 REQUIRED COMPONENTS Interpreter Development) + + execute_process( + COMMAND "${Python3_EXECUTABLE}" -c "import sysconfig; print(sysconfig.get_path('purelib'))" + OUTPUT_VARIABLE PYTHON_SITE_PACKAGES + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + message(STATUS "Found Python version: ${Python_VERSION}") + message(STATUS "Python major version: ${Python_VERSION_MAJOR}") + message(STATUS "Python minor version: ${Python_VERSION_MINOR}") + + pybind11_add_module(python_binding capi_python_binding.cpp) + target_link_libraries(python_binding PUBLIC villas) + + install( + TARGETS python_binding + COMPONENT lib + LIBRARY DESTINATION ${PYTHON_SITE_PACKAGES}/villas/node/ + ) +else() + message(STATUS "pybind11 not found. Skipping Python wrapper build.") +endif() diff --git a/python/binding/capi_python_binding.cpp b/python/binding/capi_python_binding.cpp new file mode 100644 index 000000000..66290a0a0 --- /dev/null +++ b/python/binding/capi_python_binding.cpp @@ -0,0 +1,404 @@ +/* Python-binding. + * + * Author: Kevin Vu te Laar + * SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power + * Systems, RWTH Aachen University SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +extern "C" { +#include +} + +namespace py = pybind11; + +class SamplesArray { +public: + SamplesArray(unsigned int len = 0) { + smps = (len > 0) ? new vsample *[len]() : nullptr; + this->len = len; + } + SamplesArray(const SamplesArray &) = delete; + SamplesArray &operator=(const SamplesArray &) = delete; + + ~SamplesArray() { + if (!smps) + return; + for (unsigned int i = 0; i < len; ++i) { + if (smps[i]) { + sample_decref(smps[i]); + smps[i] = nullptr; + } + } + delete[] smps; + } + + void *get_block(unsigned int start) { + return reinterpret_cast(&smps[start]); + } + + void bulk_alloc(unsigned int start_idx, unsigned int stop_idx, + unsigned int smpl_len) { + for (unsigned int i = start_idx; i < stop_idx; ++i) { + if (smps[i]) { + sample_decref(smps[i]); + } + smps[i] = sample_alloc(smpl_len); + } + } + + /* + * Performs a resize of the underlying SamplesArray copying each Sample. + * Shrinking has asymmetric behavior which may be undesired. + * Therefore use clear(). + */ + int grow(unsigned int add) { + unsigned int new_len = this->len + add; + vsample **smps_new = new vsample *[new_len](); + for (unsigned int i = 0; i < this->len; ++i) { + smps_new[i] = smps[i]; + } + delete[] smps; + this->smps = smps_new; + this->len = new_len; + + return new_len; + } + + int clear() { + if (this->smps) { + unsigned int i = 0; + for (; i < len; ++i) { + sample_decref(smps[i]); + smps[i] = nullptr; + } + delete[] smps; + smps = nullptr; + this->len = 0; + return i; + } + return -1; + } + + vsample *&operator[](unsigned int idx) { + vsample *&ref = smps[idx]; + return ref; + } + + vsample *operator[](unsigned int idx) const { return smps[idx]; } + + vsample **get_smps() { return smps; } + + unsigned int size() const { return len; } + +private: + vsample **smps; + unsigned int len; +}; + +struct timespec ns_to_timespec(int64_t time_ns) { + struct timespec ts; + ts.tv_nsec = time_ns / 1'000'000'000LL; + ts.tv_sec = time_ns % 1'000'000'000LL; + return ts; +} + +/* pybind11 can not deal with (void **) as function input parameters, + * therefore cast a simple (void *) pointer to the corresponding type + * + * wrapper bindings, sorted alphabetically + * @param villas_node Name of the module to be bound + * @param m Access variable for modifying the module code + */ +PYBIND11_MODULE(python_binding, m) { + m.def("memory_init", &memory_init); + + m.def("node_check", [](void *n) -> int { + return node_check(reinterpret_cast(n)); + }); + + m.def("node_destroy", [](void *n) -> int { + return node_destroy(reinterpret_cast(n)); + }); + + m.def( + "node_details", + [](void *n) -> const char * { + return node_details(reinterpret_cast(n)); + }, + py::return_value_policy::copy); + + m.def("node_input_signals_max_cnt", [](void *n) -> unsigned { + return node_input_signals_max_cnt(reinterpret_cast(n)); + }); + + m.def("node_is_enabled", [](void *n) -> bool { + return node_is_enabled(reinterpret_cast(n)); + }); + + m.def("node_is_valid_name", + [](const char *name) -> bool { return node_is_valid_name(name); }); + + m.def( + "node_name", + [](void *n) -> const char * { + return node_name(reinterpret_cast(n)); + }, + py::return_value_policy::copy); + + m.def( + "node_name_full", + [](void *n) -> const char * { + return node_name_full(reinterpret_cast(n)); + }, + py::return_value_policy::copy); + + m.def( + "node_name_short", + [](void *n) -> const char * { + return node_name_short(reinterpret_cast(n)); + }, + py::return_value_policy::copy); + + m.def( + "node_new", + [](const char *json_str, const char *id_str) -> vnode * { + json_error_t err; + uuid_t id; + + uuid_parse(id_str, id); + auto *json = json_loads(json_str, 0, &err); + + void *it = json_object_iter(json); + json_t *inner = json_object_iter_value(it); + + if (json_is_object(inner)) { // create node with name + return reinterpret_cast(villas::node::NodeFactory::make( + json_object_iter_value(it), id, json_object_iter_key(it))); + } else { // create node without name + char *capi_str = json_dumps(json, 0); + auto ret = node_new(id_str, capi_str); + + free(capi_str); + return ret; + } + }, + py::return_value_policy::take_ownership); + + m.def("node_output_signals_max_cnt", [](void *n) -> unsigned { + return node_output_signals_max_cnt(reinterpret_cast(n)); + }); + + m.def("node_pause", [](void *n) -> int { + return node_pause(reinterpret_cast(n)); + }); + + m.def("node_prepare", [](void *n) -> int { + return node_prepare(reinterpret_cast(n)); + }); + + m.def("node_read", [](void *n, SamplesArray &a, unsigned cnt) -> int { + return node_read(reinterpret_cast(n), a.get_smps(), cnt); + }); + + m.def("node_read", [](void *n, void *smpls, unsigned cnt) -> int { + return node_read(reinterpret_cast(n), + reinterpret_cast(smpls), cnt); + }); + + m.def("node_restart", [](void *n) -> int { + return node_restart(reinterpret_cast(n)); + }); + + m.def("node_resume", [](void *n) -> int { + return node_resume(reinterpret_cast(n)); + }); + + m.def("node_reverse", [](void *n) -> int { + return node_reverse(reinterpret_cast(n)); + }); + + m.def("node_start", [](void *n) -> int { + return node_start(reinterpret_cast(n)); + }); + + m.def("node_stop", + [](void *n) -> int { return node_stop(reinterpret_cast(n)); }); + + m.def("node_to_json_str", [](void *n) -> py::str { + auto json = reinterpret_cast(n)->toJson(); + char *json_str = json_dumps(json, 0); + auto py_str = py::str(json_str); + + json_decref(json); + free(json_str); + + return py_str; + }); + + m.def("node_write", [](void *n, SamplesArray &a, unsigned cnt) -> int { + return node_write(reinterpret_cast(n), a.get_smps(), cnt); + }); + + m.def("node_write", [](void *n, void *smpls, unsigned cnt) -> int { + return node_write(reinterpret_cast(n), + reinterpret_cast(smpls), cnt); + }); + + m.def( + "smps_array", + [](unsigned int len) -> SamplesArray * { return new SamplesArray(len); }, + py::return_value_policy::take_ownership); + + m.def("sample_alloc", + [](unsigned int len) -> vsample * { return sample_alloc(len); }); + + // Decrease reference count and release memory if last reference was held. + m.def("sample_decref", [](void *smps) -> void { + auto smp = reinterpret_cast(smps); + sample_decref(*smp); + }); + + m.def("sample_length", [](void *smp) -> unsigned { + if (smp) { + return sample_length(reinterpret_cast(smp)); + } else { + return -1; + } + }); + + m.def( + "sample_pack", + [](void *s, std::optional ts_origin_ns, + std::optional ts_received_ns) -> vsample * { + auto smp = reinterpret_cast(s); + uint64_t *seq = &smp->sequence; + unsigned len = smp->length; + double *values = reinterpret_cast(smp->data); + + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : smp->ts.origin; + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : smp->ts.received; + + return sample_pack(seq, &ts_origin, &ts_received, len, values); + }, + py::return_value_policy::reference); + + m.def( + "sample_pack", + [](const py::list values, std::optional ts_origin_ns, + std::optional ts_received_ns, unsigned seq = 0) -> void * { + struct timespec ts_origin = + ts_origin_ns ? ns_to_timespec(*ts_origin_ns) : time_now(); + struct timespec ts_received = + ts_received_ns ? ns_to_timespec(*ts_received_ns) : time_now(); + + unsigned values_len = values.size(); + double cvalues[values_len]; + + for (unsigned int i = 0; i < values_len; ++i) { + cvalues[i] = values[i].cast(); + } + uint64_t sequence = seq; + + return reinterpret_cast(sample_pack( + &sequence, &ts_origin, &ts_received, values_len, cvalues)); + }, + py::return_value_policy::reference); + + m.def( + "sample_unpack", + [](void *ss, void *ds) -> void { + auto dSmp = reinterpret_cast(ds); + auto srcSmp = reinterpret_cast(ss); + auto &destSmp = *dSmp; + + if (!srcSmp) { + throw std::runtime_error("Tried to unpack empty sample!"); + } + if (!destSmp) { + goto alloc; + } + if (destSmp->capacity < srcSmp->length) { + sample_decref(reinterpret_cast(destSmp)); + goto alloc; + } + if (0) { + alloc: + *dSmp = reinterpret_cast( + sample_alloc(srcSmp->length)); + destSmp = *dSmp; + } + + uint64_t *seq = &destSmp->sequence; + struct timespec *ts_origin = &destSmp->ts.origin; + struct timespec *ts_received = &destSmp->ts.received; + int *flags = &destSmp->flags; + unsigned *len = &destSmp->length; + double *values = reinterpret_cast(destSmp->data); + + sample_unpack(reinterpret_cast(srcSmp), seq, ts_origin, + ts_received, flags, len, values); + }, + py::return_value_policy::reference); + + m.def("sample_details", [](void *s) { + auto smp = reinterpret_cast(s); + if (!smp) { + return py::dict(); + } + + py::dict d; + d["sequence"] = smp->sequence; + d["length"] = smp->length; + d["capacity"] = smp->capacity; + d["flags"] = smp->flags; + d["refcnt"] = smp->refcnt.load(); + d["ts_origin"] = time_to_double(&smp->ts.origin); + d["ts_received"] = time_to_double(&smp->ts.received); + + py::list data; + for (unsigned int i = 0; i < smp->length; ++i) { + data.append(static_cast(smp->data[i])); + } + d["data"] = data; + + return d; + }); + + py::class_(m, "SamplesArray") + .def(py::init(), py::arg("len")) + .def("__getitem__", + [](SamplesArray &a, unsigned int idx) { + assert(idx < a.size() && "Index out of bounds"); + return a[idx]; + }) + .def("__setitem__", + [](SamplesArray &a, unsigned int idx, void *smp) { + assert(idx < a.size() && "Index out of bounds"); + if (a[idx]) { + sample_decref(a[idx]); + } + a[idx] = reinterpret_cast(smp); + }) + .def("__len__", &SamplesArray::size) + .def("bulk_alloc", &SamplesArray::bulk_alloc) + .def("grow", &SamplesArray::grow) + .def("get_block", &SamplesArray::get_block) + .def("clear", &SamplesArray::clear); +} diff --git a/python/villas/node/binding.py b/python/villas/node/binding.py new file mode 100644 index 000000000..05c0ce1f4 --- /dev/null +++ b/python/villas/node/binding.py @@ -0,0 +1,518 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +from typing import Any, Optional, Union + +import functools +import json +import logging +import weakref + +import villas.node.python_binding as vn + +Capsule = Any +logger = logging.getLogger("villas.node") + +# helper functions + + +# function decorator for optional node_compat function calls +# that would return -1 if a function is not implemented +def _warn_if_not_implemented(func): + """ + Decorator to warn if specific `node_*()` functions are not implemented. + + Returns: + Wrapping function that logs a warning if the return value is -1. + """ + + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + ret = func(self, *args, **kwargs) + if ret == -1 and hasattr(self, "_hndle") and self._hndle is not None: + msg = ( + f"[\033[93mWarning\033[0m]: Function '{func.__name__}()' " + + "is not implemented for node type " + + f"'{vn.node_name(self._hndle)}'." + ) + logger.warning(msg) + return ret + + return wrapper + + +# node API bindings + + +class Node: + class _SampleSlice: + """ + Helper class to achieve overloaded functions. + Nodes accessed via the `[]` operator, it always returns a _SampleSlice. + """ + + def __init__(self, node, idx): + self.node = weakref.proxy(node) + self.idx = idx + + def details(self): + return self.node.sample_details(self.idx) + + def read_from(self, sample_length, count=None): + return self.node.read_from(sample_length, count, self.idx) + + def write_to(self, node, count=None): + return self.node.write_to(node, count, self.idx) + + def pack_from( + self, + values: Union[float, list[float], Capsule], + ts_origin: Optional[int] = None, + ts_received: Optional[int] = None, + seq: int = 0, + ): + if isinstance(values, self.__class__): + return self.node.pack_from( + self.idx, + values.node._smps[values.idx], + ts_origin, + ts_received, + seq, + ) + else: + return self.node.pack_from(self.idx, values, ts_origin, ts_received, seq) + + def unpack_to( + self, + target: Capsule, + ): + if isinstance(target, self.__class__): + return self.node.unpack_to( + self.idx, + target.node, + target.idx, + ) + else: + raise ValueError( + "The destination must be an existing Node with an index!" + ) + + # helper functions + @staticmethod + def _ensure_capacity(smps, cap: int): + """ + Resize SamplesArray if its capacity is less than the desired capacity. + + Args: + smps: SamplesArray stored in the Node + cap (int): Desired capacity of the SamplesArray + """ + smp_cap = len(smps) + if smp_cap < cap: + smps.grow(cap - smp_cap) + + @staticmethod + def _resolve_range( + start: Optional[int], stop: Optional[int], count: Optional[int] + ) -> tuple[int, int, int]: + """ + Resolve a range dependent on start, stop and count. + At least two must be provided. + + Args: + start (int): Desired start index + stop (int): Desired stop index + count (int): Desired span of the range + + Returns: + tuple(start, stop, count) + """ + provided = sum(i is not None for i in (start, stop, count)) + if provided == 1: + raise ValueError("Two of start, stop, count must be provided") + elif provided == 2: + if start is None: + start = stop - count + if start < 0: + raise ValueError("Negative start index") + elif stop is None: + stop = start + count + else: + count = stop - start + return start, stop, count + + def __init__(self, config, uuid: Optional[str] = None, size=0): + """ + Initiallize a new node from config. + + Notes: + - Capsule is available via self._hndle. + - Capsule is available via self.config. + """ + self.config = config + self._smps = vn.smps_array(size) + if uuid is None: + self._hndle = vn.node_new(config, "") + else: + self._hndle = vn.node_new(config, uuid) + + def __del__(self): + """ + Stop and delete a node if the class object is deleted. + """ + vn.node_stop(self._hndle) + vn.node_destroy(self._hndle) + + def __getitem__(self, idx: Union[int, slice]): + """Return tuple containing self and index/slice for node operations.""" + if isinstance(idx, (int, slice)): + return Node._SampleSlice(self, idx) + else: + logger.warning("Improper array index") + raise ValueError("Improper Index") + + def __setitem__(self, obj): + if isinstance(obj, Node): + self.__del__() + self.config = obj.config + self._smps = obj._smps + else: + raise RuntimeError(f"{obj} is not of type `Node`") + + def __len__(self): + return len(self._smps) + + def __copy__(self): + """Disallow shallow copying.""" + raise RuntimeError("Copying Node is not allowed") + + def __deepcopy__(self): + """Disallow deep copying""" + raise RuntimeError("Copying a Node is not allowed") + + # bindings + @staticmethod + def memory_init(hugepages: int): + """ + Initialize internal VILLASnode memory system. + + Args: + hugepages (int): Amount of hugepages to be used. + + Notes: + - Should be called once before any memory allocation is done. + - Falls back to mmap if hugepages or root privilege unavailable. + """ + return vn.memory_init(hugepages) + + def check(self): + """Check node.""" + return vn.node_check(self._hndle) + + def details(self): + """Get node details.""" + return vn.node_details(self._hndle) + + def input_signals_max_cnt(self): + """Get max input signal count.""" + return vn.node_input_signals_max_cnt(self._hndle) + + def is_enabled(self): + """Check whether or not node is enabled.""" + return vn.node_is_enabled(self._hndle) + + @staticmethod + def is_valid_name(name: str): + """Check if a name can be used for a node.""" + return vn.node_is_valid_name(name) + + def name(self): + """Get node name.""" + return vn.node_name(self._hndle) + + def name_full(self): + """Get node name with full details.""" + return vn.node_name_full(self._hndle) + + def name_short(self): + """Get node name with less details.""" + return vn.node_name_short(self._hndle) + + def output_signals_max_cnt(self): + """Get max output signal count.""" + return vn.node_output_signals_max_cnt(self._hndle) + + def pause(self): + """Pause a node""" + return vn.node_pause(self._hndle) + + def prepare(self): + """Prepare a node""" + return vn.node_prepare(self._hndle) + + @_warn_if_not_implemented + def read_from( + self, + sample_length: int, + cnt: Optional[int] = None, + idx=None, + ): + """ + Read samples from a node into SamplesArray or block slice of samples. + + Args: + sample_length (int): Length of each sample (number of signals). + cnt (int): Number of samples to read. + + Returns: + int: Number of samples read on success or -1. + + Notes: + - Return value may vary depending on node type. + - This function may be blocking. + """ + if idx is None: + if cnt is None: + raise ValueError("Count is None") + + # resize _smps if too small + Node._ensure_capacity(self._smps, cnt) + + # allocate new samples + self._smps.bulk_alloc(0, len(self._smps), sample_length) + + return vn.node_read(self._hndle, self._smps.get_block(0), cnt) + + if isinstance(idx, int): + if cnt is None: + raise ValueError("Count is None") + start = idx + stop = start + cnt + + # if too small, resize _smps to match stop index + Node._ensure_capacity(self._smps, stop) + + # allocate new samples + self._smps.bulk_alloc(start, stop, sample_length) + + # read onward from index start + return vn.node_read(self._hndle, self._smps.get_block(start), cnt) + + elif isinstance(idx, slice): + start, stop, cnt = Node._resolve_range(idx.start, idx.stop, cnt) + + # check for length mismatch + if (stop - start) != cnt: + raise ValueError("Slice length and sample count do not match!") + # if too small, resize _smps to match stop index + Node._ensure_capacity(self._smps, stop) + + # allocate new samples + self._smps.bulk_alloc(start, stop, sample_length) + + # read onward from index start + return vn.node_read(self._hndle, self._smps.get_block(start), cnt) + + else: + logger.warning("Invalid samples Parameter") + return -1 + + def restart(self): + """Restart a node.""" + return vn.node_restart(self._hndle) + + def resume(self): + """Resume a node.""" + return vn.node_resume(self._hndle) + + @_warn_if_not_implemented + def reverse(self): + """ + Reverse node input and output. + + Notes: + - Hooks are not reversed. + - Some nodes should be stopped or restarted before reversing. + - Nodes with in-/output buffers should be stopped before reversing. + """ + return vn.node_reverse(self._hndle) + + def start(self): + """ + Start a node. + + Notes: + - Nodes are not meant to be started again without stopping first. + """ + return vn.node_start(self._hndle) + + def stop(self): + """ + Stop a node. + + Notes: + - Use before starting a node again. + - May delete in-/output buffers of a node. + """ + return vn.node_stop(self._hndle) + + def to_json(self): + """ + Return the node configuration as json object. + + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + json_str = vn.node_to_json_str(self._hndle) + json_obj = json.loads(json_str) + return json_obj + + def to_json_str(self): + """ + Returns the node configuration as string. + + Notes: + - Node configuration may not match self made configurations. + - Node configuration does not contain node name. + """ + return vn.node_to_json_str(self._hndle) + + @_warn_if_not_implemented + def write_to(self, node, cnt: Optional[int] = None, idx=None): + """ + Write samples from a SamplesArray fully or as block slice into a node. + + Args: + node: Node handle. + cnt (int): Number of samples to write. + + Returns: + int: Number of samples written on success or -1. + + Notes: + - Return value may vary depending on node type. + """ + if idx is None: + if cnt is None: + raise ValueError("Count is None") + + return vn.node_write(self._hndle, node._smps.get_block(0), cnt) + + if isinstance(idx, int): + if cnt is None: + raise ValueError("Count is None") + + start = idx + stop = start + cnt + + return vn.node_write(self._hndle, node._smps.get_block(start), cnt) + + if isinstance(idx, slice): + start, stop, _ = idx.indices(len(self._smps)) + + if cnt is None: + cnt = stop - start + + print(start, stop, cnt, len(self._smps)) + + # check for length mismatch + if (stop - start) != cnt: + raise ValueError("Slice length and sample count do not match.") + # check for out of bounds + if stop > len(self._smps): + raise IndexError("Out of bounds") + + return vn.node_write(self._hndle, node._smps.get_block(start), cnt) + + logger.warning("Invalid samples Parameter") + return -1 + + def sample_length(self, idx: int): + """Get the length of a sample.""" + if 0 <= idx and idx < len(self._smps): + return vn.sample_length(self._smps[idx]) + else: + raise IndexError(f"No Sample at index: {idx}") + + def pack_from( + self, + idx: int, + values: Union[float, list[float], Capsule], + ts_orig: Optional[int] = None, + ts_recv: Optional[int] = None, + seq: int = 0, + ): + """ + Packs a given sample from a source sample or value list. + + Args: + idx (int): Node index to store packed sample in. + ts_orig (Optional[int]): Supposed creation time in ns. + ts_recv (Optional[int]): Supposed arrival time in ns. + values (Union[float, list[float], sample]): + - Packed sample will only hold referenced values. + seq (int): supposed sequence number of the sample. + """ + if seq < 0: + raise ValueError("seq has to be positive") + + Node._ensure_capacity(self._smps, idx + 1) + if len(self._smps) <= idx: + self._smps.grow(idx + 1 - len(self._smps)) + + if isinstance(values, (float, int)): + self._smps[idx] = vn.sample_pack( + [values], + ts_orig, + ts_recv, + seq, + ) + elif isinstance(values, list): + self._smps[idx] = vn.sample_alloc(len(values)) + self._smps[idx] = vn.sample_pack(values, ts_orig, ts_recv, seq) + else: # assume a PyCapsule + self._smps[idx] = vn.sample_pack(values, ts_orig, ts_recv) + + def unpack_to( + self, + r_idx: int, + target_node, + w_idx: int, + ): + """ + Unpacks a given sample to a destined target. + + Args: + r_idx (int): Originating Node index to read from. + target_node (Node): Target node. + w_idx (int): Target Node index to unpack to. + """ + Node._ensure_capacity(self._smps, r_idx + 1) + Node._ensure_capacity(target_node._smps, w_idx + 1) + + vn.sample_unpack( + self._smps[r_idx], + target_node._smps.get_block(w_idx), + ) + + def sample_details(self, idx): + """ + Retrieve a dict with information about a sample. + + Keys: + `sequence` (int): Sequence number of the sample. + `length` (int): Sample length. + `capacity` (int): Allocated sample length. + `flags` (int): Number representing flags set of the sample. + `refcnt` (int): Reference count of the given sample. + `ts_origin` (float): Supposed timestamp of creation. + `ts_received` (float): Supposed timestamp of arrival. + + Returns: + Dict with listed keys and values. + """ + return vn.sample_details(self._smps[idx]) diff --git a/python/villas/node/binding.pyi b/python/villas/node/binding.pyi new file mode 100644 index 000000000..99980bf6e --- /dev/null +++ b/python/villas/node/binding.pyi @@ -0,0 +1,98 @@ +""" +@generated by mypy (partial). Manual edits applied for function decorators. +isort:skip_file + +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" + +from _typeshed import Incomplete +from typing import Any, Callable, Optional + +Capsule = Any +logger: Incomplete + +def _warn_if_not_implemented( + func: Callable[..., Any], +) -> Callable[..., Any]: ... + +class Node: + class _SampleSlice: + node: Incomplete + idx: Incomplete + + def __init__(self, node, idx) -> None: ... + def details(self): ... + def read_from(self, sample_length, count: Incomplete | None = None): ... + def write_to(self, node, count: Incomplete | None = None): ... + def pack_from( + self, + values: float | list[float] | Capsule, + ts_origin: int | None, + ts_received: int | None, + seq: int = 0, + ): ... + def unpack_to( + self, + target: Capsule, + ): ... + + config: Incomplete + + def __init__(self, config, uuid: str | None = None, size: int = 0) -> None: ... + def __del__(self) -> None: ... + def __getitem__(self, idx: int | slice): ... + def __setitem__(self, obj) -> None: ... + def __len__(self) -> int: ... + def __copy__(self) -> None: ... + def __deepcopy__(self) -> None: ... + @staticmethod + def memory_init(hugepages: int): ... + def check(self): ... + def details(self): ... + def input_signals_max_cnt(self): ... + def is_enabled(self): ... + @staticmethod + def is_valid_name(name: str): ... + def name(self): ... + def name_full(self): ... + def name_short(self): ... + def output_signals_max_cnt(self): ... + def pause(self): ... + def prepare(self): ... + def read_from( + self, + sample_length, + cnt: int | None = None, + idx: Incomplete | None = None, + ): ... + def restart(self): ... + def resume(self): ... + def reverse(self): ... + def start(self): ... + def stop(self): ... + def to_json(self): ... + def to_json_str(self): ... + def write_to( + self, + node, + cnt: int | None = None, + idx: Incomplete | None = None, + ): ... + def sample_length(self, idx: int): ... + def pack_from( + self, + idx: int, + ts_orig: int | None, + ts_recv: int | None, + values: float | list[float] | Capsule, + seq: int = 0, + ): ... + def unpack_to( + self, + r_idx: int, + target_node, + w_idx: int, + ): ... + def sample_details(self, idx): ... diff --git a/python/villas/node/python_binding.pyi b/python/villas/node/python_binding.pyi new file mode 100644 index 000000000..469f1e8c9 --- /dev/null +++ b/python/villas/node/python_binding.pyi @@ -0,0 +1,67 @@ +""" +@generated by mypy (partial). Manual edits applied for pybind11 binding. +isort:skip_file + +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" + +import typing +from typing import Any, Optional, overload + +capsule = Any + +class SamplesArray: + def __init__(self, len: int) -> None: ... + def bulk_alloc(self, arg0: int, arg1: int, arg2: int) -> None: ... + def clear(self) -> int: ... + def get_block(self, arg0: int) -> capsule: ... + def grow(self, arg0: int) -> int: ... + def __getitem__(self, arg0: int) -> capsule: ... + def __iter__(self) -> typing.Iterator[capsule]: ... + def __setitem__(self, arg0: int, arg1: capsule) -> None: ... + +def memory_init(arg0: int) -> int: ... +def node_check(arg0: capsule) -> int: ... +def node_destroy(arg0: capsule) -> int: ... +def node_details(arg0: capsule) -> str: ... +def node_input_signals_max_cnt(arg0: capsule) -> int: ... +def node_is_enabled(arg0: capsule) -> bool: ... +def node_is_valid_name(arg0: str) -> bool: ... +def node_name(arg0: capsule) -> str: ... +def node_name_full(arg0: capsule) -> str: ... +def node_name_short(arg0: capsule) -> str: ... +def node_new(arg0: str, arg1: str) -> capsule: ... +def node_output_signals_max_cnt(arg0: capsule) -> int: ... +def node_pause(arg0: capsule) -> int: ... +def node_prepare(arg0: capsule) -> int: ... +@overload +def node_read(arg0: capsule, arg1: SamplesArray, arg2: int) -> int: ... +@overload +def node_read(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] +def node_restart(arg0: capsule) -> int: ... +def node_resume(arg0: capsule) -> int: ... +def node_reverse(arg0: capsule) -> int: ... +def node_start(arg0: capsule) -> int: ... +def node_stop(arg0: capsule) -> int: ... +def node_to_json_str(arg0: capsule) -> str: ... +@overload +def node_write(arg0: capsule, arg1: SamplesArray, arg2: int) -> int: ... +@overload +def node_write(arg0: capsule, arg1: capsule, arg2: int) -> int: ... # type: ignore[overload-cannot-match] +def sample_alloc(arg0: int) -> capsule: ... +def sample_decref(arg0: capsule) -> None: ... +def sample_length(arg0: capsule) -> int: ... +@overload +def sample_pack(arg0: capsule, arg1: Optional[int], arg2: Optional[int]) -> None: ... +@overload +def sample_pack( + arg0: list, arg1: Optional[int], arg2: Optional[int], arg3: int +) -> capsule: ... +def sample_unpack( + arg0: capsule, + arg1: capsule, +) -> None: ... +def sample_details(arg0: capsule) -> dict: ... +def smps_array(arg0: int) -> SamplesArray: ... diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 777ea498d..ca93223da 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -18,4 +18,12 @@ add_custom_target(run-integration-tests villas-hook ) -add_dependencies(run-tests run-integration-tests) +add_custom_target(run-python-integration-tests + COMMAND + python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS + python_binding + USES_TERMINAL +) + +add_dependencies(run-tests run-integration-tests run-python-integration-tests) diff --git a/tests/integration/python/__init__.py b/tests/integration/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/python/test_binding_wrapper.py b/tests/integration/python/test_binding_wrapper.py new file mode 100644 index 000000000..29604445a --- /dev/null +++ b/tests/integration/python/test_binding_wrapper.py @@ -0,0 +1,305 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +from villas.node.binding import Node + + +class BindingWrapperIntegrationTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = Node(self.config, self.node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def test_activity_changes(self): + try: + self.test_node.check() + self.test_node.prepare() + # starting twice + self.assertEqual(0, self.test_node.start()) + + # check if the node is running + self.assertTrue(self.test_node.is_enabled()) + + # pausing twice + self.assertEqual(0, self.test_node.pause()) + self.assertEqual(-1, self.test_node.pause()) + + # resuming + self.assertEqual(0, self.test_node.resume()) + + # stopping twice + self.assertEqual(0, self.test_node.stop()) + self.assertEqual(0, self.test_node.stop()) + + # restarting + self.test_node.restart() + + # check if everything still works after restarting + self.test_node.pause() + self.test_node.resume() + self.test_node.stop() + self.test_node.start() + except Exception as e: + self.fail(f" err: {e}") + + def test_reverse_node(self): + try: + self.assertEqual(1, self.test_node.input_signals_max_cnt()) + self.assertEqual(0, self.test_node.output_signals_max_cnt()) + + self.assertEqual(0, self.test_node.reverse()) + + # input and output hooks/details are not reversed + # input and output are reversed, can be seen with wireshark and + # function test_rw_socket_and_reverse() below + self.assertEqual(1, self.test_node.input_signals_max_cnt()) + self.assertEqual(0, self.test_node.output_signals_max_cnt()) + except Exception as e: + self.fail(f"Reversing node in and output failed: {e}") + + # Test if a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string + # to create a node without a name + # uuid can not match + def test_config_from_string(self): + try: + config_str = self.test_node.to_json_str() + config_obj = json.loads(config_str) + + config_copy_str = json.dumps(config_obj, indent=2) + + test_node = Node(config_copy_str) + + self.assertEqual( + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + test_node.name_full(), + ), + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + self.test_node.name_full(), + ), + ) + except Exception as e: + self.fail(f" err: {e}") + + def test_rw_socket_and_reverse(self): + try: + data_str = json.dumps(send_recv_test, indent=2) + data = json.loads(data_str) + + test_nodes = {} + for name, content in data.items(): + obj = {name: content} + config = json.dumps(obj, indent=2) + id = str(uuid.uuid4()) + + test_nodes[name] = Node(config, id) + + for node in test_nodes.values(): + if node.check(): + raise RuntimeError("Failed to verify node configuration") + if node.prepare(): + raise RuntimeError(f"Failed to verify {node.name()} node config") + node.start() + + for i in range(100): + # Generate signals and send over send_socket + self.assertEqual(test_nodes["signal_generator"][i].read_from(2, 1), 1) + self.assertEqual( + test_nodes["send_socket"][i].write_to( + test_nodes["signal_generator"], 1 + ), + 1, + ) + self.assertEqual(test_nodes["signal_generator"].sample_length(0), 2) + + # read received signals and send them to recv_socket + self.assertEqual(test_nodes["intmdt_socket"].read_from(2, 100), 100) + self.assertEqual( + test_nodes["intmdt_socket"][:30].write_to( + test_nodes["intmdt_socket"], 30 + ), + 30, + ) + self.assertEqual( + test_nodes["intmdt_socket"][30:].write_to(test_nodes["intmdt_socket"]), + 70, + ) + # print(len(test_nodes["intmdt_socket"]._smps)) + + # confirm rev_socket signals + self.assertEqual(test_nodes["recv_socket"].read_from(2, 30), 30) + self.assertEqual(test_nodes["recv_socket"][30].read_from(2, 70), 70) + + # reversing in and outputs + # stopping the socket is necessary to clean up buffers + # starting the node again will bind the reversed socket addresses + # this can be confirmed when observing network traffic + # node details do not represent this properly as of now + for node in test_nodes.values(): + node.reverse() + node.stop() + + for node in test_nodes.values(): + node.start() + + # if another 30+70 samples are not allocated, + # sending 100 at once is impossible + self.assertEqual( + test_nodes["recv_socket"].write_to(test_nodes["recv_socket"], 100), + 100, + ) + # try writing as full slice + self.assertEqual( + test_nodes["intmdt_socket"][0:100].write_to( + test_nodes["recv_socket"], 100 + ), + 100, + ) + + except Exception as e: + self.fail(f" err: {e}") + + def test_sample_pack_unpack(self): + try: + self.test_node.pack_from(0, [0.01, 1.01, 2.01, 3.01, 4.01]) + self.test_node[1].pack_from( + [1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100 + ) + self.test_node[2].pack_from(42, int(1e9), int(1e9) + 100) + self.test_node[3].pack_from(self.test_node[1], int(1e9), int(1e9) + 100) + self.test_node[2].unpack_to(self.test_node[1]) + self.assertEqual([42.0], self.test_node[1].details()["data"]) + self.test_node[0].unpack_to(self.test_node[1]) + self.assertEqual( + [0.01, 1.01, 2.01, 3.01, 4.01], + self.test_node[1].details()["data"], + ) + self.test_node[0].unpack_to(self.test_node[2]) + self.test_node[0].unpack_to(self.test_node[4]) + self.test_node[1].unpack_to(self.test_node[2]) + except Exception as e: + self.fail(f"err: {e}") + + def test_samplesarray_size(self): + try: + node_config = json.dumps(test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + node = Node(node_config, node_uuid, 100) + self.assertEqual(len(node), 100) + node[199].pack_from([1.01, 2.01, 3.01, 4.01, 5.01], int(1e9), int(1e9) + 100) + self.assertEqual(len(node), 200) + node[199].unpack_to(node[299]) + self.assertEqual(len(node), 300) + except Exception as e: + self.fail(f"err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +send_recv_test = { + "send_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65532", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65533", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "intmdt_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65533", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65534", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "recv_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65534", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65535", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "signal_generator": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + }, +} + +if __name__ == "__main__": + unittest.main() diff --git a/tests/integration/python/test_python_binding.py b/tests/integration/python/test_python_binding.py new file mode 100644 index 000000000..bb20c3550 --- /dev/null +++ b/tests/integration/python/test_python_binding.py @@ -0,0 +1,287 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +import villas.node.python_binding as pb + + +class BindingIntegrationTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = pb.node_new(self.config, self.node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def tearDown(self): + try: + pb.node_stop(self.test_node) + pb.node_destroy(self.test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_activity_changes(self): + try: + pb.node_check(self.test_node) + pb.node_prepare(self.test_node) + # starting twice + self.assertEqual(0, pb.node_start(self.test_node)) + + # check if the node is running + self.assertTrue(pb.node_is_enabled(self.test_node)) + + # pausing twice + self.assertEqual(0, pb.node_pause(self.test_node)) + self.assertEqual(-1, pb.node_pause(self.test_node)) + + # resuming + self.assertEqual(0, pb.node_resume(self.test_node)) + + # stopping twice + self.assertEqual(0, pb.node_stop(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) + + # restarting + pb.node_restart(self.test_node) + + # check if everything still works after restarting + pb.node_pause(self.test_node) + pb.node_resume(self.test_node) + pb.node_stop(self.test_node) + pb.node_start(self.test_node) + except Exception as e: + self.fail(f" err: {e}") + + def test_reverse_node(self): + try: + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) + + self.assertEqual(0, pb.node_reverse(self.test_node)) + + # input and output hooks/details are not reversed + # input and output are reversed, can be seen with wireshark and + # function test_rw_socket_and_reverse() below + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"Reversing node in and output failed: {e}") + + # Test if a node can be recreated with the string from node_to_json_str + # node_to_json_str has a wrong config format causing the config string + # to create a node without a name + # uuid can not match + def test_config_from_string(self): + try: + config_str = pb.node_to_json_str(self.test_node) + config_obj = json.loads(config_str) + + config_copy_str = json.dumps(config_obj, indent=2) + + test_node = pb.node_new(config_copy_str, "") + + self.assertEqual( + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + pb.node_name_full(test_node), + ), + re.sub( + r"^[^:]+: uuid=[0-9a-fA-F-]+, ", + "", + pb.node_name_full(self.test_node), + ), + ) + except Exception as e: + self.fail(f" err: {e}") + + def test_rw_socket_and_reverse(self): + try: + data_str = json.dumps(send_recv_test, indent=2) + data = json.loads(data_str) + + test_nodes = {} + for name, content in data.items(): + obj = {name: content} + config = json.dumps(obj, indent=2) + id = str(uuid.uuid4()) + + test_nodes[name] = pb.node_new(config, id) + + for node in test_nodes.values(): + if pb.node_check(node): + raise RuntimeError("Failed to verify node configuration") + if pb.node_prepare(node): + raise RuntimeError( + f"Failed to verify {pb.node_name(node)} node config" + ) + pb.node_start(node) + + # Arrays to store samples + send_smpls = pb.smps_array(1) + intmdt_smpls = pb.smps_array(100) + recv_smpls = pb.smps_array(100) + + for i in range(100): + # send_smpls holds a new sample each time, but the + # old one still has a reference in the socket buffer (below) + # it is necessary to allocate a new sample each time + send_smpls[0] = pb.sample_alloc(2) + intmdt_smpls[i] = pb.sample_alloc(2) + recv_smpls[i] = pb.sample_alloc(2) + + # Generate signals and send over send_socket + self.assertEqual( + pb.node_read(test_nodes["signal_generator"], send_smpls, 1), + 1, + ) + self.assertEqual( + pb.node_write(test_nodes["send_socket"], send_smpls, 1), 1 + ) + + # read received signals and send them to recv_socket + self.assertEqual( + pb.node_read(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, + ) + self.assertEqual( + pb.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, + ) + + # confirm rev_socket signals + self.assertEqual( + pb.node_read(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) + + # reversing in and outputs + # stopping the socket is necessary to clean up buffers + # starting the node again will bind the reversed socket addresses + # this can be confirmed when observing network traffic + # node details do not represent this properly as of now + for node in test_nodes.values(): + pb.node_reverse(node) + pb.node_stop(node) + + for node in test_nodes.values(): + pb.node_start(node) + + self.assertEqual( + pb.node_write(test_nodes["recv_socket"], recv_smpls, 100), 100 + ) + self.assertEqual( + pb.node_write(test_nodes["intmdt_socket"], intmdt_smpls, 100), + 100, + ) + + # cleanup + for node in test_nodes.values(): + pb.node_stop(node) + pb.node_destroy(node) + + except Exception as e: + self.fail(f" err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +send_recv_test = { + "send_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65532", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65533", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "intmdt_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65533", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65534", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "recv_socket": { + "type": "socket", + "format": "protobuf", + "layer": "udp", + "in": { + "address": "127.0.0.1:65534", + "signals": [ + {"name": "voltage", "type": "float", "unit": "V"}, + {"name": "current", "type": "float", "unit": "A"}, + ], + }, + "out": { + "address": "127.0.0.1:65535", + "netem": {"enabled": False}, + "multicast": {"enabled": False}, + }, + }, + "signal_generator": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + }, +} + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f9098e53e..44af84ea9 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -35,5 +35,13 @@ add_custom_target(run-unit-tests USES_TERMINAL ) +add_custom_target(run-python-unit-tests + COMMAND + python3 -m unittest discover ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS + python_binding + USES_TERMINAL +) + add_dependencies(tests unit-tests) -add_dependencies(run-tests run-unit-tests) +add_dependencies(run-tests run-unit-tests run-python-unit-tests) diff --git a/tests/unit/python/__init__.py b/tests/unit/python/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/python/test_binding_wrapper.py b/tests/unit/python/test_binding_wrapper.py new file mode 100644 index 000000000..b88b3eeab --- /dev/null +++ b/tests/unit/python/test_binding_wrapper.py @@ -0,0 +1,241 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +from villas.node.binding import Node + + +class BindingWrapperUnitTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = Node(self.config, self.node_uuid) + config = json.dumps(signal_test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + self.signal_test_node = Node(config, node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def test_start(self): + try: + self.assertEqual(0, self.test_node.start()) + self.assertEqual(0, self.signal_test_node.start()) + except Exception as e: + self.fail(f"err: {e}") + + @unittest.skip( + """Starting a socket twice will result in a RuntimeError. + Thise will leave the socket IP bound and may mess with other tests. + The behavior is Node specific.""" + ) + def test_start_err(self): + try: + self.assertEqual(0, self.test_node.start()) + with self.assertRaises((AssertionError, RuntimeError)): + self.test_node.start() + except Exception as e: + self.fail(f"err: {e}") + + def test_new(self): + try: + node_config = json.dumps(test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + node = Node(node_config, node_uuid) + self.assertIsNotNone(node) + except Exception as e: + self.fail(f"err: {e}") + + def test_check(self): + try: + self.test_node.check() + except Exception as e: + self.fail(f"err: {e}") + + def test_prepare(self): + try: + self.test_node.prepare() + except Exception as e: + self.fail(f"err: {e}") + + def test_is_enabled(self): + try: + self.assertTrue(self.test_node.is_enabled()) + except Exception as e: + self.fail(f"err: {e}") + + def test_pause(self): + try: + self.assertEqual(-1, self.test_node.pause()) + self.assertEqual(-1, self.test_node.pause()) + except Exception as e: + self.fail(f"err: {e}") + + def test_resume(self): + try: + self.assertEqual(0, self.test_node.resume()) + except Exception as e: + self.fail(f"err: {e}") + + def test_stop(self): + try: + self.assertEqual(0, self.test_node.start()) + self.assertEqual(0, self.test_node.stop()) + self.assertEqual(0, self.test_node.stop()) + except Exception as e: + self.fail(f"err: {e}") + + def test_restart(self): + try: + self.assertEqual(0, self.test_node.restart()) + self.assertEqual(0, self.test_node.restart()) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name(self): + try: + # remove color codes before checking for equality + self.assertEqual( + "test_node(socket)", + re.sub(r"\x1b\[[0-9;]*m", "", self.test_node.name()), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_short(self): + try: + self.assertEqual("test_node", self.test_node.name_short()) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_full(self): + try: + node = self.test_node + self.assertEqual( + "test_node(socket)" + + ": uuid=" + + self.node_uuid + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + re.sub(r"\x1b\[[0-9;]*m", "", node.name_full()), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_details(self): + try: + self.assertEqual( + "layer=udp, " + + "in.address=0.0.0.0:12000, " + + "out.address=127.0.0.1:12001", + self.test_node.details(), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_to_json(self): + try: + if not isinstance(self.test_node.to_json(), dict): + self.fail("Not a JSON object (dict)") + except Exception as e: + self.fail(f"err: {e}") + + def test_node_to_json_str(self): + try: + json.loads(self.test_node.to_json_str()) + except Exception as e: + self.fail(f"err: {e}") + + def test_input_signals_max_cnt(self): + try: + self.assertEqual(1, self.test_node.input_signals_max_cnt()) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_output_signals_max_cnt(self): + try: + self.assertEqual(0, self.test_node.output_signals_max_cnt()) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_is_valid_name(self): + try: + invalid_names = [ + "", + "###", + "v@l:d T3xt w;th invalid symb#ls", + "33_characters_long_string_invalid", + ] + valid_names = ["32_characters_long_strings_valid", "valid_name"] + + for name in invalid_names: + self.assertFalse(Node.is_valid_name(name)) + for name in valid_names: + self.assertTrue(Node.is_valid_name(name)) + except Exception as e: + self.fail(f"err: {e}") + + def test_reverse(self): + try: + # socket has reverse() implemented, expected return 0 + self.assertEqual(0, self.test_node.reverse()) + self.assertEqual(0, self.test_node.reverse()) + + # signal.v2 has not reverse() implemented, expected return 1 + self.assertEqual(-1, self.signal_test_node.reverse()) + self.assertEqual(-1, self.signal_test_node.reverse()) + except Exception as e: + self.fail(f"err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +signal_test_node_config = { + "signal_test_node": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + } +} + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/python/test_python_binding.py b/tests/unit/python/test_python_binding.py new file mode 100644 index 000000000..1ded31c73 --- /dev/null +++ b/tests/unit/python/test_python_binding.py @@ -0,0 +1,239 @@ +""" +Author: Kevin Vu te Laar +SPDX-FileCopyrightText: 2014-2025 Institute for Automation of Complex Power Systems, RWTH Aachen University +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import json +import re +import unittest +import uuid +import villas.node.python_binding as pb + + +class BindingUnitTests(unittest.TestCase): + def setUp(self): + try: + self.config = json.dumps(test_node_config, indent=2) + self.node_uuid = str(uuid.uuid4()) + self.test_node = pb.node_new(self.config, self.node_uuid) + config = json.dumps(signal_test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + self.signal_test_node = pb.node_new(config, node_uuid) + except Exception as e: + self.fail(f"new_node err: {e}") + + def tearDown(self): + try: + pb.node_stop(self.test_node) + pb.node_destroy(self.test_node) + pb.node_stop(self.signal_test_node) + pb.node_destroy(self.signal_test_node) + except Exception as e: + self.fail(f"node cleanup error: {e}") + + def test_start(self): + try: + self.assertEqual(0, pb.node_start(self.test_node)) + self.assertEqual(0, pb.node_start(self.signal_test_node)) + except Exception as e: + self.fail(f"err: {e}") + + @unittest.skip( + """ + Starting a socket twice will result in a RuntimeError. + Thise will leave the socket IP bound and may mess with other tests. + The behavior is Node specific. + """ + ) + def test_start_err(self): + try: + self.assertEqual(0, pb.node_start(self.test_node)) + with self.assertRaises((AssertionError, RuntimeError)): + pb.node_start(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_new(self): + try: + node_config = json.dumps(test_node_config, indent=2) + node_uuid = str(uuid.uuid4()) + node = pb.node_new(node_config, node_uuid) + self.assertIsNotNone(node) + except Exception as e: + self.fail(f"err: {e}") + + def test_check(self): + try: + pb.node_check(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_prepare(self): + try: + pb.node_prepare(self.test_node) + except Exception as e: + self.fail(f"err: {e}") + + def test_is_enabled(self): + try: + self.assertTrue(pb.node_is_enabled(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_pause(self): + try: + self.assertEqual(-1, pb.node_pause(self.test_node)) + self.assertEqual(-1, pb.node_pause(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_resume(self): + try: + self.assertEqual(0, pb.node_resume(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_stop(self): + try: + self.assertEqual(0, pb.node_start(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) + self.assertEqual(0, pb.node_stop(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_restart(self): + try: + self.assertEqual(0, pb.node_restart(self.test_node)) + self.assertEqual(0, pb.node_restart(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name(self): + try: + # remove color codes before checking for equality + self.assertEqual( + "test_node(socket)", + re.sub(r"\x1b\[[0-9;]*m", "", pb.node_name(self.test_node)), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_short(self): + try: + self.assertEqual("test_node", pb.node_name_short(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_name_full(self): + try: + node = self.test_node + self.assertEqual( + "test_node(socket)" + + ": uuid=" + + self.node_uuid + + ", #in.signals=1/1, #in.hooks=0, #out.hooks=0" + + ", in.vectorize=1, out.vectorize=1, out.netem=no, layer=udp" + + ", in.address=0.0.0.0:12000, out.address=127.0.0.1:12001", + re.sub(r"\x1b\[[0-9;]*m", "", pb.node_name_full(node)), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_details(self): + try: + self.assertEqual( + "layer=udp, " + + "in.address=0.0.0.0:12000, " + + "out.address=127.0.0.1:12001", + pb.node_details(self.test_node), + ) + except Exception as e: + self.fail(f"err: {e}") + + def test_input_signals_max_cnt(self): + try: + self.assertEqual(1, pb.node_input_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_output_signals_max_cnt(self): + try: + self.assertEqual(0, pb.node_output_signals_max_cnt(self.test_node)) + except Exception as e: + self.fail(f"err: {e}") + + def test_node_is_valid_name(self): + try: + invalid_names = [ + "", + "###", + "v@l:d T3xt w;th invalid symb#ls", + "33_characters_long_string_invalid", + ] + valid_names = ["32_characters_long_strings_valid", "valid_name"] + + for name in invalid_names: + self.assertFalse(pb.node_is_valid_name(name)) + for name in valid_names: + self.assertTrue(pb.node_is_valid_name(name)) + except Exception as e: + self.fail(f"err: {e}") + + def test_reverse(self): + try: + # socket has reverse() implemented, expected return 0 + self.assertEqual(0, pb.node_reverse(self.test_node)) + self.assertEqual(0, pb.node_reverse(self.test_node)) + + # signal.v2 has not reverse() implemented, expected return 1 + self.assertEqual(-1, pb.node_reverse(self.signal_test_node)) + self.assertEqual(-1, pb.node_reverse(self.signal_test_node)) + except Exception as e: + self.fail(f"err: {e}") + + +test_node_config = { + "test_node": { + "type": "socket", + "format": "villas.binary", + "layer": "udp", + "in": { + "address": "*:12000", + "signals": [{"name": "tap_position", "type": "integer", "init": 0}], + }, + "out": {"address": "127.0.0.1:12001"}, + } +} + +signal_test_node_config = { + "signal_test_node": { + "type": "signal.v2", + "limit": 100, + "rate": 10, + "in": { + "signals": [ + { + "amplitude": 2, + "name": "voltage", + "phase": 90, + "signal": "sine", + "type": "float", + "unit": "V", + }, + { + "amplitude": 1, + "name": "current", + "phase": 0, + "signal": "sine", + "type": "float", + "unit": "A", + }, + ], + "hooks": [{"type": "print", "format": "villas.human"}], + }, + } +} + +if __name__ == "__main__": + unittest.main()