diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 11914a459..0b097746b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -162,8 +162,8 @@ test:python: script: - cd python - pytest --verbose . - - black --extend-exclude=".*(\\.pyi|_pb2.py)$" --check . - - flake8 --extend-exclude="*.pyi,*_pb2.py" . + - black --line-length=90 --extend-exclude=".*(\\.pyi|_pb2.py)$" --check . + - flake8 --max-line-length=90 --extend-exclude="*.pyi,*_pb2.py" . - mypy . image: ${DOCKER_IMAGE_DEV}:${DOCKER_TAG} needs: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65e169643..71fbea36b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -50,6 +50,16 @@ repos: hooks: - id: black-jupyter exclude: .*_pb2.pyi?$ + args: + - --line-length=90 + + - repo: https://github.com/pycqa/flake8 + rev: "7.3.0" + hooks: + - id: flake8 + exclude: .*_pb2.pyi?$ + args: + - --max-line-length=90 - repo: https://github.com/markdownlint/markdownlint rev: "v0.13.0" diff --git a/clients/python/client.py b/clients/python/client.py index b6e543d17..81c3de52a 100644 --- a/clients/python/client.py +++ b/clients/python/client.py @@ -1,9 +1,15 @@ # -*- coding: utf-8 -*- -# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University +# SPDX-FileCopyrightText: 2014-2023 Institute for Automation of +# Complex Power Systems, RWTH Aachen University # SPDX-License-Identifier: Apache-2.0 import villas_pb2 -import time, socket, errno, sys, os, signal +import time +import socket +import errno +import sys +import os +import signal layer = sys.argv[1] if len(sys.argv) == 2 else "udp" @@ -50,6 +56,7 @@ # Gracefully shutdown def sighandler(signum, frame): + global running running = False diff --git a/etc/examples/nodes/opal_orchestra.conf b/etc/examples/nodes/opal_orchestra.conf index f7693e9ce..f319e495a 100644 --- a/etc/examples/nodes/opal_orchestra.conf +++ b/etc/examples/nodes/opal_orchestra.conf @@ -7,7 +7,7 @@ nodes = { domain1 = { type = "opal.orchestra" - # Path to the OPAL-RT Orchestra Data Defintion XML file (DDF). + # Path to the OPAL-RT Orchestra Data Definition XML file (DDF). ddf = "orchestra.xml" # Enable to overwrite the DDF file. @@ -45,12 +45,12 @@ nodes = { # For 'local' extcomm = "udp"; - addr_framework = "10.168.13.5"; + addr_framework = "127.0.0.1"; port_framework = 10000 core_framework = 0 core_client = 0 - nic_framework = "eno2" - nic_client = "eno1" + nic_framework = "eth0" + nic_client = "eth0" # For 'remote' card = "test" @@ -92,7 +92,7 @@ nodes = { out = { signals = ( - { name="pub_signal_float", init = 1.2, orchestra_name = "sub_signal_float", type = "float" } + { name="sub_signal_float", init = 1.2, orchestra_name = "sub_signal_float", type = "float" } ) } } diff --git a/etc/python/example.py b/etc/python/example.py index 6be8b623e..0cb950fc1 100644 --- a/etc/python/example.py +++ b/etc/python/example.py @@ -9,7 +9,8 @@ villas node <(python3 etc/python/example.py) Author: Steffen Vogel - SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + SPDX-FileCopyrightText: 2014-2023 Institute for Automation of + Complex Power Systems, RWTH Aachen University SPDX-License-Identifier: Apache-2.0 """ @@ -38,7 +39,10 @@ "type": "socket", "layer": "udp", "format": "protobuf", - "in": {"address": "*:12000", "signals": [{"name": "in", "type": "float"}]}, + "in": { + "address": "*:12000", + "signals": [{"name": "in", "type": "float"}], + }, "out": {"address": f"5.6.7.8:{port}"}, } diff --git a/flake.nix b/flake.nix index 22f15e472..5f4563838 100644 --- a/flake.nix +++ b/flake.nix @@ -90,8 +90,13 @@ # Cross-compiled packages - villas-node-x86_64-linux = if pkgs.system == "x86_64-linux" then pkgs.villas-node else pkgs.pkgsCross.x86_64-linux.villas-node; - villas-node-aarch64-linux = if pkgs.system == "aarch64-linux" then pkgs.villas-node else pkgs.pkgsCross.aarch64-multiplatform.villas-node; + villas-node-x86_64-linux = + if pkgs.system == "x86_64-linux" then pkgs.villas-node else pkgs.pkgsCross.x86_64-linux.villas-node; + villas-node-aarch64-linux = + if pkgs.system == "aarch64-linux" then + pkgs.villas-node + else + pkgs.pkgsCross.aarch64-multiplatform.villas-node; dockerImage-x86_64-linux = pkgs.dockerTools.buildLayeredImage { name = "villas-node"; @@ -206,7 +211,10 @@ villas = { imports = [ (nixDir + "/module.nix") ]; - nixpkgs.overlays = [ self.overlays.default self.overlays.patches ]; + nixpkgs.overlays = [ + self.overlays.default + self.overlays.patches + ]; }; }; }; diff --git a/include/villas/nodes/example.hpp b/include/villas/nodes/example.hpp deleted file mode 100644 index c821bdde4..000000000 --- a/include/villas/nodes/example.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/* An example get started with new implementations of new node-types. - * - * This example does not do any particularly useful. - * It is just a skeleton to get you started with new node-types. - * - * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include -#include -#include - -namespace villas { -namespace node { - -// Forward declarations -struct Sample; - -class ExampleNode : public Node { - -protected: - // Place any configuration and per-node state here - - // Settings - int setting1; - - std::string setting2; - - // States - int state1; - struct timespec start_time; - - int _read(struct Sample *smps[], unsigned cnt) override; - int _write(struct Sample *smps[], unsigned cnt) override; - -public: - ExampleNode(const uuid_t &id = {}, const std::string &name = ""); - - /* All of the following virtual-declared functions are optional. - * Have a look at node.hpp/node.cpp for the default behaviour. - */ - - virtual ~ExampleNode(); - - int prepare() override; - - int parse(json_t *json) override; - - int check() override; - - int start() override; - - // int stop() override; - - // int pause() override; - - // int resume() override; - - // int restart() override; - - // int reverse() override; - - // std::vector getPollFDs() override; - - // std::vector getNetemFDs() override; - - // struct villas::node::memory::Type *getMemoryType() override; - - const std::string &getDetails() override; -}; - -} // namespace node -} // namespace villas diff --git a/lib/nodes/example.cpp b/lib/nodes/example.cpp index 0c16b7917..56b969370 100644 --- a/lib/nodes/example.cpp +++ b/lib/nodes/example.cpp @@ -9,148 +9,159 @@ */ #include -#include -#include #include #include +#include #include using namespace villas; -using namespace villas::node; using namespace villas::utils; +using namespace villas::node; + +class ExampleNode : public Node { + +protected: + // Place any configuration and per-node state here + + // Settings + int setting1; + + std::string setting2; + + // States + int state1; + struct timespec start_time; + + int _read(struct Sample *smps[], unsigned cnt) override { + int read; + struct timespec now; -ExampleNode::ExampleNode(const uuid_t &id, const std::string &name) - : Node(id, name), setting1(72), setting2("something"), state1(0) {} + // TODO: Add implementation here. The following is just an example -ExampleNode::~ExampleNode() {} + assert(cnt >= 1 && smps[0]->capacity >= 1); -int ExampleNode::prepare() { - state1 = setting1; + now = time_now(); - if (setting2 == "double") - state1 *= 2; + smps[0]->data[0].f = time_delta(&now, &start_time); - return 0; -} + /* Dont forget to set other flags in struct Sample::flags + * E.g. for sequence no, timestamps... */ + smps[0]->flags = (int)SampleFlags::HAS_DATA; + smps[0]->signals = getInputSignals(false); -int ExampleNode::parse(json_t *json) { - // TODO: Add implementation here. The following is just an example + read = 1; // The number of samples read - const char *setting2_str = nullptr; + return read; + } - json_error_t err; - int ret = json_unpack_ex(json, &err, 0, "{ s?: i, s?: s }", "setting1", - &setting1, "setting2", &setting2_str); - if (ret) - throw ConfigError(json, err, "node-config-node-example"); + int _write(struct Sample *smps[], unsigned cnt) override { + int written; - if (setting2_str) - setting2 = setting2_str; + // TODO: Add implementation here. - return 0; -} + written = 0; // The number of samples written -int ExampleNode::check() { - if (setting1 > 100 || setting1 < 0) - return -1; + return written; + } - if (setting2.empty() || setting2.size() > 10) - return -1; +public: + ExampleNode(const uuid_t &id = {}, const std::string &name = "") + : Node(id, name), setting1(72), setting2("something"), state1(0) {} - return Node::check(); -} + /* All of the following virtual-declared functions are optional. + * Have a look at node.hpp/node.cpp for the default behaviour. + */ -int ExampleNode::start() { - // TODO add implementation here + virtual ~ExampleNode() {} - start_time = time_now(); + int prepare() override { + state1 = setting1; - return 0; -} + if (setting2 == "double") + state1 *= 2; -// int ExampleNode::stop() -// { -// // TODO add implementation here -// return 0; -// } + return 0; + } -// int ExampleNode::pause() -// { -// // TODO add implementation here -// return 0; -// } + int parse(json_t *json) override { + // TODO: Add implementation here. The following is just an example -// int ExampleNode::resume() -// { -// // TODO add implementation here -// return 0; -// } + const char *setting2_str = nullptr; -// int ExampleNode::restart() -// { -// // TODO add implementation here -// return 0; -// } + json_error_t err; + int ret = json_unpack_ex(json, &err, 0, "{ s?: i, s?: s }", "setting1", + &setting1, "setting2", &setting2_str); + if (ret) + throw ConfigError(json, err, "node-config-node-example"); -// int ExampleNode::reverse() -// { -// // TODO add implementation here -// return 0; -// } + if (setting2_str) + setting2 = setting2_str; -// std::vector ExampleNode::getPollFDs() -// { -// // TODO add implementation here -// return {}; -// } + return 0; + } -// std::vector ExampleNode::getNetemFDs() -// { -// // TODO add implementation here -// return {}; -// } + int check() override { + if (setting1 > 100 || setting1 < 0) + return -1; -// struct villas::node::memory::Type * ExampleNode::getMemoryType() -// { -// // TODO add implementation here -// } + if (setting2.empty() || setting2.size() > 10) + return -1; -const std::string &ExampleNode::getDetails() { - details = fmt::format("setting1={}, setting2={}", setting1, setting2); - return details; -} + return Node::check(); + } -int ExampleNode::_read(struct Sample *smps[], unsigned cnt) { - int read; - struct timespec now; + const std::string &getDetails() override { + details = fmt::format("setting1={}, setting2={}", setting1, setting2); + return details; + } - // TODO: Add implementation here. The following is just an example + int start() override { + // TODO add implementation here - assert(cnt >= 1 && smps[0]->capacity >= 1); + start_time = time_now(); - now = time_now(); + return 0; + } - smps[0]->data[0].f = time_delta(&now, &start_time); + // int ExampleNode::stop() override { + // // TODO add implementation here + // return 0; + // } - /* Dont forget to set other flags in struct Sample::flags - * E.g. for sequence no, timestamps... */ - smps[0]->flags = (int)SampleFlags::HAS_DATA; - smps[0]->signals = getInputSignals(false); + // int ExampleNode::pause() override { + // // TODO add implementation here + // return 0; + // } - read = 1; // The number of samples read + // int ExampleNode::resume() override { + // // TODO add implementation here + // return 0; + // } - return read; -} + // int ExampleNode::restart() override { + // // TODO add implementation here + // return 0; + // } -int ExampleNode::_write(struct Sample *smps[], unsigned cnt) { - int written; + // int ExampleNode::reverse() override { + // // TODO add implementation here + // return 0; + // } - // TODO: Add implementation here. + // std::vector ExampleNode::getPollFDs() override { + // // TODO add implementation here + // return {}; + // } - written = 0; // The number of samples written + // std::vector ExampleNode::getNetemFDs() override { + // // TODO add implementation here + // return {}; + // } - return written; -} + // struct villas::node::memory::Type * ExampleNode::getMemoryType() override { + // // TODO add implementation here + // } +}; // Register node static char n[] = "example"; diff --git a/lib/nodes/opal_orchestra.cpp b/lib/nodes/opal_orchestra.cpp index 0bb804927..cba9a456e 100644 --- a/lib/nodes/opal_orchestra.cpp +++ b/lib/nodes/opal_orchestra.cpp @@ -7,8 +7,8 @@ */ #include +#include #include -#include #include #include @@ -20,6 +20,8 @@ extern "C" { #include + +__attribute__((weak)) void op_license_feature_register() {} } #include @@ -235,13 +237,15 @@ class OpalOrchestraMapping { class OpalOrchestraNode : public Node { protected: - Task task; // The task which is used to pace the node in asynchronous mode. + // The task which is used to pace the node in asynchronous mode. + Task task; - unsigned int - connectionKey; // A connection key identifies a connection between a specific combo of Orchestra's framework and client. + // A connection key identifies a connection between a specific combo of Orchestra's framework and client. + unsigned int connectionKey; unsigned int *status; - Domain domain; // The domain to which the node belongs. + // The domain to which the node belongs. + Domain domain; std::unordered_map, OpalOrchestraMapping> subscribeMappings; @@ -252,14 +256,21 @@ class OpalOrchestraNode : public Node { std::optional dataDefinitionFilename; std::chrono::seconds connectTimeout; - std::optional - flagDelay; // Define a delay to wait, this will call the system function usleep and free the CPU. - std::optional - flagDelayTool; // Force the local Orchestra communication to be made with flag instead of semaphore when using an external communication process. - bool skipWaitToGo; // Skip wait-to-go step during start. - bool dataDefinitionFileOverwrite; // Overwrite the data definition file (DDF). - bool - dataDefinitionFileWriteOnly; // Overwrite the data definition file (DDF) and terminate VILLASnode. + + // Define a delay to wait, this will call the system function usleep and free the CPU. + std::optional flagDelay; + + // Force the local Orchestra communication to be made with flag instead of semaphore when using an external communication process. + std::optional flagDelayTool; + + // Skip wait-to-go step during start. + bool skipWaitToGo; + + // Overwrite the data definition file (DDF). + bool dataDefinitionFileOverwrite; + + // Overwrite the data definition file (DDF) and terminate VILLASnode. + bool dataDefinitionFileWriteOnly; int _read(struct Sample *smps[], unsigned cnt) override { if (dataDefinitionFileWriteOnly) { @@ -706,8 +717,7 @@ class OpalOrchestraNodeFactory : public NodeFactory { int getFlags() const override { return (int)NodeFactory::Flags::SUPPORTS_READ | (int)NodeFactory::Flags::SUPPORTS_WRITE | - (int)NodeFactory::Flags::SUPPORTS_POLL | - (int)NodeFactory::Flags::HIDDEN; + (int)NodeFactory::Flags::SUPPORTS_POLL; } std::string getName() const override { return "opal.orchestra"; } @@ -715,6 +725,25 @@ class OpalOrchestraNodeFactory : public NodeFactory { std::string getDescription() const override { return "OPAL-RT Orchestra client"; } + + int start(SuperNode *sn) override { + const std::string opalBin = "/usr/opalrt/common/bin"; + + // Append /usr/opalrt/common/bin to PATH on OPAL-RT Linux systems + // for finding OrchestraExtComm tools. + std::string path = getenv("PATH"); + + if (path.find(opalBin) == std::string::npos) { + path = path + ":" + opalBin; + + auto ret = setenv("PATH", path.c_str(), 1); + if (ret != 0) { + throw RuntimeError("Failed to set PATH environment variable"); + } + } + + return 0; + } }; static OpalOrchestraNodeFactory p; diff --git a/packaging/nix/orchestra.nix b/packaging/nix/orchestra.nix index 5d0a839e3..b9e736a65 100644 --- a/packaging/nix/orchestra.nix +++ b/packaging/nix/orchestra.nix @@ -2,35 +2,81 @@ # SPDX-License-Identifier: Apache-2.0 { + version ? "2025.1.3", + + lib, stdenv, + autoPatchelfHook, + makeWrapper, + runCommand, + fetchurl, + libredirect, nettools, - fetchurl, - autoPatchelfHook, - dpkg, libuuid, - makeWrapper , + unzip, + rpm, + cpio, + cacert, + jq, + curl, + gnused, }: let - # src = requireFile { - # name = "componentorchestra_7.6.2_amd64.deb"; - # hash = "sha256-2cQtYkf1InKrDPL5UDQDHaYM7bq21Dw77itfFwuXa54="; - # }; - - src = fetchurl { - url = "https://blob.opal-rt.com/softwares/rt-lab-archives/componentorchestra_7.6.2_amd64.deb?sp=r&st=2024-10-30T06:31:59Z&se=2034-11-30T14:31:59Z&spr=https&sv=2022-11-02&sr=b&sig=cnKY8RxZf8hv91gWLIBG6iBGSVziXkKR3%2BOYIE6MSkI%3D"; - hash = "sha256-2cQtYkf1InKrDPL5UDQDHaYM7bq21Dw77itfFwuXa54="; + inherit (lib) + concatStringsSep + escapeURL + mapAttrsToList + replaceStrings + ; + + versionHashes = { + "2025.1.2" = "sha256-XZp5lprMwBAst3LTIVYoOfUpk1p66EJ4mY/nYPdBfIE="; + "2025.1.3" = "sha256-ISoRxPUHE1KpdvXfLyFujLixTtfcG6jRJoKgMwIwynY="; }; + + blobName = "RT-LAB_${version}.zip"; + + megainstaller-zip = + runCommand blobName + { + outputHash = versionHashes.${version}; + outputHashAlgo = null; + + buildInputs = [ + cacert + curl + jq + gnused + ]; + } + '' + SAS=$(curl -s https://www.opal-rt.com/wp-json/wp/v2/pages/2058 | jq -r .content.rendered | sed -En 's|.*https://blob\.opal-rt\.com/softwares/[^?]+\?([^"]*).*|\1|p' | sed 's|&|\&|g' | head -1) + curl -o $out "https://blob.opal-rt.com/softwares/RT-LAB/${blobName}?$SAS" + ''; + + megainstaller = runCommand "rtlab-megainstaller-${version}" { inherit version; } '' + ${unzip}/bin/unzip ${megainstaller-zip} -d $out + ''; + + target = runCommand "rtlab-target-${version}" { inherit version; } '' + ${unzip}/bin/unzip ${megainstaller}/Files/RT-LAB/data/target.zip + mv target $out + ''; + + target_rpm = runCommand "rtlab-target-${version}-rpm" { inherit version; } '' + mkdir $out + cd $out + + ${rpm}/bin/rpm2cpio ${target}/rt_linux64/rtlab-rt_linux64-*.rpm | ${cpio}/bin/cpio -idmv + ''; in stdenv.mkDerivation { pname = "libOpalOrchestra"; - version = "7.6.2"; - inherit src; - - dontUnpack = true; + inherit version; + src = target_rpm; nativeBuildInputs = [ - dpkg autoPatchelfHook makeWrapper ]; @@ -41,10 +87,14 @@ stdenv.mkDerivation { ]; installPhase = '' - ${dpkg}/bin/dpkg-deb -x ${src} . + mkdir -p $out/{lib,bin,include} + + cd usr/opalrt/v2025.1.3.77/common - mv usr/opalrt/exportedOrchestra $out - mv $out/bin/OrchestraExtCommIPDebian $out/bin/OrchestraExtCommIP + cp "bin/OrchestraExtCommIP" "$out/bin/" + cp "include_target/RTAPI.h" "$out/include/" + cp "bin/libsimulation-configuration.so" "$out/lib/" + cp "bin/libOpalOrchestra.so" "$out/lib/libOpalOrchestra.so" ''; preFixup = '' diff --git a/packaging/nix/patches.nix b/packaging/nix/patches.nix index 72f04359a..796f67c6c 100644 --- a/packaging/nix/patches.nix +++ b/packaging/nix/patches.nix @@ -10,7 +10,7 @@ let in { libiec61850 = prev.libiec61850.overrideAttrs { - patches = [ ./libiec61850_debug_r_session.patch ]; + patches = [ ../patches/libiec61850_debug_r_session.patch ]; cmakeFlags = (prev.cmakeFlags or [ ]) ++ [ "-DCONFIG_USE_EXTERNAL_MBEDTLS_DYNLIB=ON" "-DCONFIG_EXTERNAL_MBEDTLS_DYNLIB_PATH=${final.mbedtls}/lib" diff --git a/packaging/nix/villas.nix b/packaging/nix/villas.nix index 74492896c..246150a67 100644 --- a/packaging/nix/villas.nix +++ b/packaging/nix/villas.nix @@ -200,6 +200,11 @@ stdenv.mkDerivation { ++ lib.optionals withNodeInfiniband [ rdma-core ] ++ lib.optionals withExtraConfig [ libconfig ]; + preFixup = lib.optional withNodeOpalOrchestra '' + wrapProgram $out/bin/villas-node \ + --prefix PATH : ${orchestra}/bin/ + ''; + meta = { mainProgram = "villas"; description = "a tool connecting real-time power grid simulation equipment"; diff --git a/packaging/nix/libiec61850_debug_r_session.patch b/packaging/patches/libiec61850_debug_r_session.patch similarity index 100% rename from packaging/nix/libiec61850_debug_r_session.patch rename to packaging/patches/libiec61850_debug_r_session.patch diff --git a/python/pyproject.toml b/python/pyproject.toml index 882af94cf..debd26a8b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -12,7 +12,7 @@ description = 'Python support for the VILLASnode simulation-data gateway' readme = 'README.md' requires-python = '>=3.10' keywords = ['simulation', 'power', 'system', 'real-time', 'villas'] -license = 'Apache-2.0' +# license = 'Apache-2.0' classifiers = [ 'Development Status :: 4 - Beta', 'Topic :: Scientific/Engineering', @@ -21,18 +21,20 @@ classifiers = [ ] dependencies = [ 'linuxfd==1.5; platform_system=="Linux"', - 'requests==2.32.3', 'protobuf==6.31.1', + 'libconf==2.0.1', ] optional-dependencies.dev = [ 'black==25.1.0', 'flake8==7.2.0', 'mypy==1.15.0', 'pytest==8.3.5', - 'types-requests==2.32.0.20250328', 'types-protobuf==5.29.1.20250403', ] +[project.scripts] +villas-conf2orchestra-ddf = "villas.node.opal_orchestra_ddf:main" + [project.urls] GitHub = 'https://github.com/VILLASframework/node' Project = 'https://www.fein-aachen.org/en/projects/villas-node' @@ -50,8 +52,14 @@ email = 'Philipp.Jungkamp@opal-rt.com' module = ['google'] ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ['libconf'] +ignore_missing_imports = true + + [tool.black] extend-exclude = '''.*(\.pyi|_pb2.py)$''' +line-length = 90 [tool.setuptools.packages.find] include = ["villas.node"] diff --git a/python/villas/node/formats.py b/python/villas/node/formats.py index c99c14952..c5e7b355a 100644 --- a/python/villas/node/formats.py +++ b/python/villas/node/formats.py @@ -156,8 +156,7 @@ def load_sample(self, sample: str) -> Sample | None: return None m = re.match( - r"(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?" - r"(?:e[+-]?\d+)?)?(?:\((\d+)\))?(F)?", + r"(\d+)(?:\.(\d+))?([-+]\d+(?:\.\d+)?" r"(?:e[+-]?\d+)?)?(?:\((\d+)\))?(F)?", fields[0], ) diff --git a/python/villas/node/node.py b/python/villas/node/node.py index 183624868..e17740dd1 100644 --- a/python/villas/node/node.py +++ b/python/villas/node/node.py @@ -11,8 +11,8 @@ import signal import subprocess from tempfile import NamedTemporaryFile - -import requests +import urllib.request +import urllib.error LOGGER = logging.getLogger("villas.node") @@ -126,15 +126,25 @@ def load_config(self, i): self.request("restart", method="POST", json=req) def request(self, action, method="GET", **args): - if "timeout" not in args: - args["timeout"] = 1 + timeout = args.get("timeout", 1) + json_data = args.get("json") - r = requests.request( - method, f"{self.api_url}/api/{self.api_version}/{action}", **args - ) - r.raise_for_status() + url = f"{self.api_url}/api/{self.api_version}/{action}" + + # Prepare the request + req = urllib.request.Request(url, method=method) + + # Add JSON data if provided + if json_data: + req.add_header("Content-Type", "application/json") + data = json.dumps(json_data).encode("utf-8") + req.data = data - return r.json() + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as e: + raise e def get_local_version(self): ver = subprocess.check_output([self.executable, "-V"]) diff --git a/python/villas/node/opal_orchestra_ddf.py b/python/villas/node/opal_orchestra_ddf.py new file mode 100644 index 000000000..3e76d4135 --- /dev/null +++ b/python/villas/node/opal_orchestra_ddf.py @@ -0,0 +1,281 @@ +""" +Author: Steffen Vogel +SPDX-FileCopyrightText: 2025 OPAL-RT Germany GmbH +SPDX-License-Identifier: Apache-2.0 +""" # noqa: E501 + +import argparse +import sys +import xml.etree.ElementTree as ET +from xml.dom import minidom + +try: + import libconf +except ImportError: + # In case we use a local copy of libconf + from . import libconf # type: ignore + + +def generate_ddf(node_cfg): + """ + Generate an OPAL-RT Orchestra DDF XML file from a VILLASnode configuration. + + Args: + node_cfg: Dictionary containing the node configuration + from VILLASnode config + + Returns: + str: XML-encoded string representing the Orchestra DDF + """ + orchestra = ET.Element("orchestra") + + domain = ET.SubElement(orchestra, "domain", name=node_cfg.domain) + + # Add domain properties + synchronous = node_cfg.get("synchronous", False) + ET.SubElement(domain, "synchronous").text = "yes" if synchronous else "no" + + multiple_publish = node_cfg.get("multiple_publish_allowed", False) + ET.SubElement(domain, "multiplePublishAllowed").text = ( + "yes" if multiple_publish else "no" + ) + + states = node_cfg.get("states", False) + ET.SubElement(domain, "states").text = "yes" if states else "no" + + # Add connection element + conn_cfg = node_cfg.get("connection", {}) + add_connection(domain, conn_cfg) + + # Process input signals -> PUBLISH set + in_cfg = node_cfg.get("in", {}) + in_signals = in_cfg.get("signals", []) + + if in_signals: + publish_set = ET.SubElement(domain, "set", name="PUBLISH") + add_signals_to_set(publish_set, in_signals, is_publish=False) + + # Process output signals -> SUBSCRIBE set + out_cfg = node_cfg.get("out", {}) + out_signals = out_cfg.get("signals", []) + + if out_signals: + subscribe_set = ET.SubElement(domain, "set", name="SUBSCRIBE") + add_signals_to_set(subscribe_set, out_signals, is_publish=True) + + # Convert to pretty-printed XML string + xml_str = ET.tostring(orchestra, encoding="unicode") + dom = minidom.parseString(xml_str) + pretty_xml = dom.toprettyxml(indent=" ", encoding="UTF-8") + + return pretty_xml.decode("utf-8") + + +def add_connection(parent_elem, conn_cfg): + """ + Add a connection element to the parent XML element based + on connection configuration. + + Args: + parent_elem: ET.Element to add the connection to + conn_cfg: Dictionary containing connection configuration + """ + conn_type = conn_cfg.get("type", "local") + + if conn_type == "local": + conn_elem = ET.SubElement( + parent_elem, + "connection", + **{ + "type": "local", + "extcomm": conn_cfg.get("extcomm", "udp"), + "addrframework": conn_cfg.get("addr_framework", ""), + "portframework": str(conn_cfg.get("port_framework", 10000)), + "nicframework": conn_cfg.get("nic_framework", ""), + "nicclient": conn_cfg.get("nic_client", ""), + "coreframework": str(conn_cfg.get("core_framework", 0)), + "coreclient": str(conn_cfg.get("core_client", 0)), + }, + ) + + elif conn_type == "remote": + conn_elem = ET.SubElement(parent_elem, "connection", type="remote") + + card_elem = ET.SubElement(conn_elem, "card") + card_elem.text = conn_cfg.get("card", "") + + pciindex_elem = ET.SubElement(conn_elem, "pciindex") + pciindex_elem.text = str(conn_cfg.get("pci_index", 0)) + + elif conn_type == "dolphin": + conn_elem = ET.SubElement(parent_elem, "connection", type="dolphin") + + nodeidframework_elem = ET.SubElement(conn_elem, "nodeIdFramework") + nodeidframework_elem.text = str(conn_cfg.get("node_id_framework", 0)) + + segmentid_elem = ET.SubElement(conn_elem, "segmentId") + segmentid_elem.text = str(conn_cfg.get("segment_id", 0)) + + +def add_signals_to_set(parent_elem, signals, is_publish=True): + """ + Add signals to a PUBLISH or SUBSCRIBE set element. + Handles nested bus structures based on orchestra_name paths. + + Args: + parent_elem: ET.Element to add signals to + signals: List of signal configurations + is_publish: Boolean indicating if this is for + PUBLISH (True) or SUBSCRIBE (False) + """ + # Group signals by their orchestra names and build nested structure + signal_tree = {} + + for sig in signals: + orchestra_name = sig.get("orchestra_name", sig.name) + orchestra_type = sig.get("orchestra_type", "float64") + orchestra_index = sig.get("orchestra_index") + + # Split by '/' to handle nested buses + orchestra_name_parts = orchestra_name.split("/") + + # Build nested structure + current = signal_tree + + for i, part in enumerate(orchestra_name_parts): + is_leaf = i == len(orchestra_name_parts) - 1 + + if part not in current: + current[part] = {"_signal": None, "_children": {}} + + if is_leaf: + signal = current[part]["_signal"] + if signal is None: + if orchestra_index is None: + orchestra_index = 0 + + current[part]["_signal"] = { + "name": part, + "type": orchestra_type, + "length": orchestra_index + 1, + "default": sig.get("init") if not is_publish else None, + } + else: + if orchestra_type != signal["type"]: + raise RuntimeError( + "Conflicting definitions for signal " + f"'{orchestra_name}'" + ) + + index = orchestra_index + if index is None: + index = signal["length"] + + if index >= signal["length"]: + signal["length"] = index + 1 + + else: + # Navigate to next level + current = current[part]["_children"] + + # Build XML from tree + build_xml_from_tree(parent_elem, signal_tree, is_publish) + + +def build_xml_from_tree(parent_elem, tree, is_publish): + """ + Recursively build XML elements from the signal tree. + + Args: + parent_elem: Parent XML element + tree: Nested dictionary representing signal hierarchy + is_publish: Boolean for PUBLISH vs SUBSCRIBE + """ + for name, node in sorted(tree.items()): + signal = node.get("_signal", None) + children = node.get("_children", {}) + + if signal: + # Determine if this is a bus (has children) or a simple signal + is_bus = len(children) > 0 + + item = ET.SubElement(parent_elem, "item", name=signal["name"]) + + if is_bus: + ET.SubElement(item, "type").text = "bus" + # Recursively add children + build_xml_from_tree(item, children, is_publish) + else: + ET.SubElement(item, "type").text = signal["type"] + length = signal.get("length", 1) + ET.SubElement(item, "length").text = str(length) + + # Add default for SUBSCRIBE + if not is_publish: + default_val = signal.get("default") + if default_val is not None: + if signal["type"] == "boolean": + default_text = "yes" if default_val else "no" + else: + default_text = str(default_val) + else: + if signal["type"] == "boolean": + default_text = "no" + else: + default_text = "0" + ET.SubElement(item, "default").text = default_text + elif children: + # This is a bus without direct signals + item = ET.SubElement(parent_elem, "item", name=name) + ET.SubElement(item, "type").text = "bus" + build_xml_from_tree(item, children, is_publish) + + +def parse_args(): + parser = argparse.ArgumentParser( + description="VILLASnode OPAL Orchestra configuration generator" + ) + parser.add_argument( + "--villas-config", + "-i", + type=str, + help="Path to VILLASnode configuration file", + ) + parser.add_argument( + "--orchestra-ddf", + "-o", + type=str, + help="Path to output OPAL-RT Orchestra DDF file", + ) + parser.add_argument( + "node", + type=str, + help="Name of the node to generate OPAL-RT Orchestra DDF for", + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + if args.villas_config is None: + villas_cfg = libconf.load(sys.stdin) + else: + with open(args.villas_config) as f: + villas_cfg = libconf.load(f) + + node_cfg = villas_cfg.nodes.get(args.node) + if node_cfg is None: + raise RuntimeError(f"Node '{args.node}' not found in configuration") + + ddf = generate_ddf(node_cfg) + + if args.orchestra_ddf is None: + sys.stdout.write(ddf) + else: + with open(args.orchestra_ddf, "w") as f: + f.write(ddf) + + +if __name__ == "__main__": + main() diff --git a/python/villas/node/villas_pb2.py b/python/villas/node/villas_pb2.py index a3aa59453..b764662c0 100644 --- a/python/villas/node/villas_pb2.py +++ b/python/villas/node/villas_pb2.py @@ -22,8 +22,6 @@ _sym_db = _symbol_database.Default() - - DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cvillas.proto\x12\x0bvillas.node\"/\n\x07Message\x12$\n\x07samples\x18\x01 \x03(\x0b\x32\x13.villas.node.Sample\"\xfe\x01\n\x06Sample\x12,\n\x04type\x18\x01 \x02(\x0e\x32\x18.villas.node.Sample.Type:\x04\x44\x41TA\x12\x10\n\x08sequence\x18\x02 \x01(\x04\x12)\n\tts_origin\x18\x03 \x01(\x0b\x32\x16.villas.node.Timestamp\x12+\n\x0bts_received\x18\x04 \x01(\x0b\x32\x16.villas.node.Timestamp\x12\x11\n\tnew_frame\x18\x05 \x01(\x08\x12\"\n\x06values\x18\x64 \x03(\x0b\x32\x12.villas.node.Value\"%\n\x04Type\x12\x08\n\x04\x44\x41TA\x10\x01\x12\t\n\x05START\x10\x02\x12\x08\n\x04STOP\x10\x03\"&\n\tTimestamp\x12\x0b\n\x03sec\x18\x01 \x02(\r\x12\x0c\n\x04nsec\x18\x02 \x02(\r\"Z\n\x05Value\x12\x0b\n\x01\x66\x18\x01 \x01(\x01H\x00\x12\x0b\n\x01i\x18\x02 \x01(\x03H\x00\x12\x0b\n\x01\x62\x18\x03 \x01(\x08H\x00\x12!\n\x01z\x18\x04 \x01(\x0b\x32\x14.villas.node.ComplexH\x00\x42\x07\n\x05value\"%\n\x07\x43omplex\x12\x0c\n\x04real\x18\x01 \x02(\x02\x12\x0c\n\x04imag\x18\x02 \x02(\x02') _globals = globals() diff --git a/tests/benchmarks/evaluate_logs.ipynb b/tests/benchmarks/evaluate_logs.ipynb index 907322a49..3047336b1 100644 --- a/tests/benchmarks/evaluate_logs.ipynb +++ b/tests/benchmarks/evaluate_logs.ipynb @@ -126,7 +126,9 @@ "\n", " # Match settings, as described above\n", " matchObj = re.match(\n", - " r\".*?(\\d*)_(\\w*)-(\\d*)-(\\d*)-(\\d*)_output.csv\", file, re.M | re.I\n", + " r\".*?(\\d*)_(\\w*)-(\\d*)-(\\d*)-(\\d*)_output.csv\",\n", + " file,\n", + " re.M | re.I,\n", " )\n", "\n", " # Fill values to array\n", @@ -145,9 +147,7 @@ " if re.match(r\".*?_input.csv\", file, re.M | re.I):\n", " # Load file\n", " input_dataset[i].append(\n", - " np.genfromtxt(\n", - " \"{}/{}/{}\".format(rootdir, subdir, file), delimiter=\",\"\n", - " )\n", + " np.genfromtxt(\"{}/{}/{}\".format(rootdir, subdir, file), delimiter=\",\")\n", " )\n", "\n", " print(\"Loaded input dataset from: {}\".format(file))\n", @@ -155,9 +155,7 @@ " # Regex to match output files files\n", " elif re.match(r\".*?_output.csv\", file, re.M | re.I):\n", " output_dataset[i].append(\n", - " np.genfromtxt(\n", - " \"{}/{}/{}\".format(rootdir, subdir, file), delimiter=\",\"\n", - " )\n", + " np.genfromtxt(\"{}/{}/{}\".format(rootdir, subdir, file), delimiter=\",\")\n", " )\n", "\n", " print(\"Loaded output dataset from: {}\".format(file))\n", @@ -363,10 +361,16 @@ "\n", " # Take percentage\n", " perc_never_trans_total_arr[i].append(\n", - " round(never_trans_total_arr[i][j] / int(settings_array[i][j][4]) * 100, 2)\n", + " round(\n", + " never_trans_total_arr[i][j] / int(settings_array[i][j][4]) * 100,\n", + " 2,\n", + " )\n", " )\n", " perc_never_trans_after_arr[i].append(\n", - " round(never_trans_after_arr[i][j] / int(settings_array[i][j][4]) * 100, 2)\n", + " round(\n", + " never_trans_after_arr[i][j] / int(settings_array[i][j][4]) * 100,\n", + " 2,\n", + " )\n", " )\n", "\n", " print(\n", @@ -545,7 +549,11 @@ " color=\"#00549f\",\n", " )\n", " ax.axvline(\n", - " medians[i][j], color=\"red\", linestyle=\"-\", linewidth=1, alpha=0.85\n", + " medians[i][j],\n", + " color=\"red\",\n", + " linestyle=\"-\",\n", + " linewidth=1,\n", + " alpha=0.85,\n", " )\n", "\n", " # Set axis and calculate values above limit\n", @@ -639,7 +647,8 @@ " never_trans_total_arr[i][j], perc_never_trans_total_arr[i][j]\n", " )\n", " never_transferred_text += \"while connected: {0:5d} ({1:5.2f}%)\".format(\n", - " never_trans_after_arr[i][j], perc_never_trans_after_arr[i][j]\n", + " never_trans_after_arr[i][j],\n", + " perc_never_trans_after_arr[i][j],\n", " )\n", "\n", " # Set font properties for headers and text\n", @@ -732,7 +741,11 @@ "\n", " fig.savefig(\n", " \"{}/{}_{}_{}i_{}j.pdf\".format(\n", - " rootdir, settings_array[i][j][0], settings_array[i][j][2], i, j\n", + " rootdir,\n", + " settings_array[i][j][0],\n", + " settings_array[i][j][2],\n", + " i,\n", + " j,\n", " ),\n", " format=\"pdf\",\n", " )\n", @@ -929,7 +942,8 @@ " for k in range(0, len(settings[\"3d_plot\"][\"ticks\"][\"y\"])):\n", " for l in range(0, len(settings[\"3d_plot\"][\"ticks\"][\"x\"])):\n", " Z = np.append(\n", - " Z, medians[i][k * len(settings[\"3d_plot\"][\"ticks\"][\"x\"]) + l]\n", + " Z,\n", + " medians[i][k * len(settings[\"3d_plot\"][\"ticks\"][\"x\"]) + l],\n", " )\n", "\n", " ###################################\n", diff --git a/tools/hwdef-parse.py b/tools/hwdef-parse.py index 95ff13656..b3ca7c7ec 100755 --- a/tools/hwdef-parse.py +++ b/tools/hwdef-parse.py @@ -2,16 +2,17 @@ """ HWH File Parser -Author: Steffen Vogel -Author: Daniel Krebs -Author: Hatim Kanchwala -Author: Pascal Bauer -Author: Niklas Eiling +Author: Daniel Krebs +Author: Hatim Kanchwala +Author: Pascal Bauer +Author: Niklas Eiling SPDX-FileCopyrightText: 2017-2022 Daniel Krebs SPDX-FileCopyrightText: 2017-2022 Hatim Kanchwala SPDX-FileCopyrightText: 2023 Pascal Bauer -SPDX-FileCopyrightText: 2024 Niklas Eiling +SPDX-FileCopyrightText: 2024 Niklas Eiling + SPDX-License-Identifier: GPL-3.0-or-later This program is free software: you can redistribute it and/or modify @@ -28,7 +29,6 @@ along with this program. If not, see . """ -from multiprocessing.sharedctypes import Value from lxml import etree import zipfile import sys @@ -63,8 +63,9 @@ ["acs.eonerc.rwth-aachen.de", "sysgen"], ] -# List of VLNI ids of AXI4-Stream infrastructure IP cores which do not alter data -# see PG085 (AXI4-Stream Infrastructure IP Suite v2.2) +# List of VLNI ids of AXI4-Stream infrastructure IP cores +# which do not alter data see +# PG085 (AXI4-Stream Infrastructure IP Suite v2.2) axi_converter_whitelist = [ ["xilinx.com", "ip", "axis_subset_converter"], ["xilinx.com", "ip", "axis_clock_converter"], @@ -86,9 +87,8 @@ def bus_trace(root, busname, type, whitelist): module = root.xpath( - './/MODULE[.//BUSINTERFACE[@BUSNAME="{}" and (@TYPE="{}" or @TYPE="{}")]]'.format( - busname, type[0], type[1] - ) + './/MODULE[.//BUSINTERFACE[@BUSNAME="{}" '.format(busname) + + 'and (@TYPE="{}" or @TYPE="{}")]]'.format(type[0], type[1]) ) vlnv = module[0].get("VLNV") @@ -120,7 +120,7 @@ def vlnv_match(vlnv, whitelist): def remove_prefix(text, prefix): - return text[text.startswith(prefix) and len(prefix) :] + return text[text.startswith(prefix) and len(prefix) :] # noqa: E203 def sanitize_name(name): @@ -138,23 +138,23 @@ def sanitize_name(name): sys.exit(1) try: - # read .hwdef which is actually a zip-file + # Read .hwdef which is actually a zip-file zip = zipfile.ZipFile(sys.argv[1], "r") hwh = zip.read("top.hwh") -except: +except Exception: f = open(sys.argv[1], "r") hwh = f.read() -# parse .hwh file which is actually XML +# Parse .hwh file which is actually XML try: root = etree.XML(hwh) -except: +except Exception: print('Bad format of "{}"! Did you choose the right file?'.format(sys.argv[1])) sys.exit(1) ips = {} -# find all whitelisted modules +# Find all whitelisted modules modules = root.find(".//MODULES") for module in modules: instance = module.get("INSTANCE") @@ -166,11 +166,11 @@ def sanitize_name(name): ips[instance] = {"vlnv": vlnv} - # populate parameters + # Populate parameters params = module.find(".//PARAMETERS") if ( params is not None and instance != "zynq_ultra_ps_e_0" - ): #! Parameters of "zynq" ignored + ): # Parameters of "zynq" ignored p = ips[instance].setdefault("parameters", {}) for param in params: @@ -186,7 +186,7 @@ def sanitize_name(name): p[name] = value - # populate memory view + # Populate memory view mmap = module.find(".//MEMORYMAP") if mmap is None: continue @@ -206,7 +206,7 @@ def sanitize_name(name): _block["size"] = _block["highaddr"] - _block["baseaddr"] + 1 -# find AXI-Stream switch port mapping +# Find AXI-Stream switch port mapping switch = root.find('.//MODULE[@MODTYPE="axis_switch"]') busifs = switch.find(".//BUSINTERFACES") switch_ports = 0 @@ -254,13 +254,10 @@ def sanitize_name(name): } ) -# set number of master/slave port pairs for switch +# Set number of master/slave port pairs for switch ips[switch.get("INSTANCE")]["num_ports"] = int(switch_ports / 2) -# find interrupt assignments - - -# find interrupt assignments +# Find interrupt assignments intr_controllers = [] intr_signals = [] @@ -281,9 +278,8 @@ def sanitize_name(name): for intc, intr in zip(intr_controllers, intr_signals): concat = root.xpath( - './/MODULE[@MODTYPE="xlconcat" and .//PORT[@SIGNAME="{}" and @DIR="O"]]'.format( - intr - ) + './/MODULE[@MODTYPE="xlconcat" ' + + 'and .//PORT[@SIGNAME="{}" and @DIR="O"]]'.format(intr) )[0] ports = concat.xpath('.//PORT[@DIR="I"]') @@ -299,9 +295,9 @@ def sanitize_name(name): m = r.search(name) irq = int(m.group(1)) - ip = root.xpath( - './/MODULE[.//PORT[@SIGNAME="{}" and @DIR="O"]]'.format(signame) - )[0] + ip = root.xpath('.//MODULE[.//PORT[@SIGNAME="{}" and @DIR="O"]]'.format(signame))[ + 0 + ] instance = ip.get("INSTANCE") vlnv = ip.get("VLNV") @@ -309,11 +305,12 @@ def sanitize_name(name): originators = [] - # follow one level of OR gates merging interrupts (may be generalized later) + # Follow one level of OR gates merging interrupts + # (may be generalized later) if modtype == "util_vector_logic": logic_op = ip.xpath('.//PARAMETER[@NAME="C_OPERATION"]')[0] if logic_op.get("VALUE") == "or": - # hardware interrupts sharing the same IRQ at the controller + # Hardware interrupts sharing the same IRQ at the controller ports = ip.xpath('.//PORT[@DIR="I"]') for port in ports: signame = port.get("SIGNAME") @@ -323,7 +320,7 @@ def sanitize_name(name): instance = ip.get("INSTANCE") originators.append((instance, signame)) else: - # consider this instance as originator + # Consider this instance as originator originators.append((instance, signame)) for instance, signame in originators: @@ -361,7 +358,7 @@ def sanitize_name(name): ("PCIEBAR", "AXIBAR", pcie_bars), ): barnum = pcie.find('.//PARAMETER[@NAME="C_{}_NUM"]'.format(from_bar)) - if barnum == None: + if barnum is None: barnum = pcie.find('.//PARAMETER[@NAME="{}_NUM"]'.format(from_bar)) from_bar_num = int(barnum.get("VALUE"))