diff --git a/.gitignore b/.gitignore index 4d7b589e..d04a76a1 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,5 @@ Icon? ehthumbs.db Thumbs.db .cvsignore +.vs +.vscode \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..d2b7ea2b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "VideoData/resource/cmake/rticonnextdds-cmake-utils"] + path = VideoData/resource/cmake/rticonnextdds-cmake-utils + url = https://github.com/rticommunity/rticonnextdds-cmake-utils.git +[submodule "AppTelemetry/resource/cmake/rticonnextdds-cmake-utils"] + path = AppTelemetry/resource/cmake/rticonnextdds-cmake-utils + url = https://github.com/rticommunity/rticonnextdds-cmake-utils.git diff --git a/AppTelemetry/CMakeLists.txt b/AppTelemetry/CMakeLists.txt new file mode 100644 index 00000000..2d944d3a --- /dev/null +++ b/AppTelemetry/CMakeLists.txt @@ -0,0 +1,108 @@ +# (c) 2025 Copyright, Real-Time Innovations, Inc. All rights reserved. +# RTI grants Licensee a license to use, modify, compile, and create derivative +# works of the Software. Licensee has the right to distribute object form only +# for use with RTI products. The Software is provided "as is", with no warranty +# of any type, including any warranty for fitness for any purpose. RTI is under +# no obligation to maintain or support the Software. RTI shall not be liable +# for any incidental or consequential damages arising out of the use or +# inability to use the software. + +cmake_minimum_required(VERSION 3.10) +project(oteladapter) + + +# Find RTI Connext dependencies +list(APPEND CMAKE_MODULE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/resource/cmake/rticonnextdds-cmake-utils/cmake/Modules" +) + +include(ConnextDdsGenerateFiles) +include(ConnextDdsCodegen) + +# Find RTI Connext DDS +find_package( + RTIConnextDDS "6.1.2" + REQUIRED + COMPONENTS + core + routing_service +) + +find_package(CURL REQUIRED) + +# Use find_package to include OpenTelemetry C++ requirements +find_package(opentelemetry-cpp CONFIG REQUIRED + PATHS "${CMAKE_CURRENT_SOURCE_DIR}/otel-cpp/lib/cmake/opentelemetry-cpp/" +) +find_package(prometheus-cpp CONFIG REQUIRED + PATHS "${CMAKE_CURRENT_SOURCE_DIR}/otel-cpp/lib/cmake/prometheus-cpp/" +) + +# Include directories +include_directories( + ${RTIConnextDDS_INCLUDE_DIRS} + "${CMAKE_CURRENT_SOURCE_DIR}/otel-cpp/include" +) + +add_library(${PROJECT_NAME} SHARED + "${CMAKE_CURRENT_SOURCE_DIR}/src/oteladapter.cxx" + "${CMAKE_CURRENT_SOURCE_DIR}/src/otelconnection.cxx" + "${CMAKE_CURRENT_SOURCE_DIR}/src/otelstreamwriter.cxx" +) + +set_target_properties(${PROJECT_NAME} + PROPERTIES + CXX_STANDARD 11 + CXX_STANDARD_REQUIRED ON + LIBRARY_OUTPUT_DIRECTORY "${output_dir}" + LIBRARY_OUTPUT_DIRECTORY_RELEASE "${output_dir}" + LIBRARY_OUTPUT_DIRECTORY_DEBUG "${output_dir}" + ARCHIVE_OUTPUT_DIRECTORY "${output_dir}" + ARCHIVE_OUTPUT_DIRECTORY_RELEASE "${output_dir}" + ARCHIVE_OUTPUT_DIRECTORY_DEBUG "${output_dir}" + RUNTIME_OUTPUT_DIRECTORY "${output_dir}" + RUNTIME_OUTPUT_DIRECTORY_RELEASE "${output_dir}" + RUNTIME_OUTPUT_DIRECTORY_DEBUG "${output_dir}" +) + +target_link_libraries(${PROJECT_NAME} + RTIConnextDDS::routing_service_infrastructure + RTIConnextDDS::cpp2_api + ${CONNEXTDDS_EXTERNAL_LIBS} + ${OPENTELEMETRY_CPP_LIBRARIES} +) + +# Generator application +connextdds_rtiddsgen_run( + IDL_FILE + "${CMAKE_CURRENT_SOURCE_DIR}/generator/telemetry.idl" + OUTPUT_DIRECTORY + "${CMAKE_CURRENT_BINARY_DIR}/src/c++11" + LANG C++11 + VAR "METRICS_TYPE" + DISABLE_PREPROCESSOR +) + +add_executable(generator + "${CMAKE_CURRENT_SOURCE_DIR}/generator/c++11/generator.cxx" + ${METRICS_TYPE_CXX11_SOURCES} + "${CMAKE_CURRENT_SOURCE_DIR}/generator/c++11/application.hpp" + ${METRICS_TYPE_CXX11_HEADERS} +) +target_link_libraries(generator + PRIVATE + RTIConnextDDS::cpp2_api +) +target_include_directories(generator + PRIVATE + "${CMAKE_CURRENT_BINARY_DIR}/src/c++11" +) + +connextdds_copy_files( + INPUT_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/src/RsTelemetryGateway.xml" + "${CMAKE_CURRENT_SOURCE_DIR}/src/run_tmux.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/generator/c++11/generator_qos.xml" + OUTPUT_DIR + "${CMAKE_CURRENT_BINARY_DIR}/" +) diff --git a/AppTelemetry/Dockerfile b/AppTelemetry/Dockerfile new file mode 100644 index 00000000..8d3b406d --- /dev/null +++ b/AppTelemetry/Dockerfile @@ -0,0 +1,120 @@ +# (c) 2025 Copyright, Real-Time Innovations, Inc. All rights reserved. +# RTI grants Licensee a license to use, modify, compile, and create derivative +# works of the Software. Licensee has the right to distribute object form only +# for use with RTI products. The Software is provided "as is", with no warranty +# of any type, including any warranty for fitness for any purpose. RTI is under +# no obligation to maintain or support the Software. RTI shall not be liable +# for any incidental or consequential damages arising out of the use or +# inability to use the software. + + +# The Dockerfile is based on the official Ubuntu image for the jammy release. +# The CONNEXT_VERSION argument is used to specify the version of RTI Connext to install. +# The RTI_LICENSE_AGREEMENT_ACCEPTED argument is used to accept the RTI license agreement during installation. +# The NDDSHOME environment variable is set to the installation directory of RTI Connext. +# +# The Dockerfile installs the requisite RTI Connext Debian Packages from the official RTI repository. It also installs some build tools and the license file. +# +# The CMD instruction specifies the default command to run when the container starts. In this case, it runs the /bin/bash shell. +# +# To build the Docker image, run the following command from the root repository folder: +# docker build -t connext:oteladapter --build-arg RTI_LICENSE_AGREEMENT_ACCEPTED=accepted --build-arg CONNEXT_VERSION=7.3.0 . +# +# To run the Docker container, run the following command: +# docker run -it --rm --net=host -v $RTI_LICENSE_FILE:/root/rti_license.dat connext:oteladapter +# +# The -it option is used to run the container in interactive mode. +# The --rm option is used to remove the container when it exits. +# +# The container will start and run the /bin/bash shell. + +FROM ubuntu:jammy AS install-stage + +ARG CONNEXT_VERSION=7.3.0 +ARG RTI_LICENSE_AGREEMENT_ACCEPTED + +ENV DISPLAY=:0 +ENV SHELL=/bin/bash +# CHANGE THE FOLLOWING LINE TO MATCH YOUR TIMEZONE +ENV TZ=Europe/Madrid +ENV NDDSHOME=/opt/rti.com/rti_connext_dds-${CONNEXT_VERSION} + +# Install the required packages +RUN export DEBIAN_FRONTEND=noninteractive \ + && apt-get update \ + && apt-get install -y \ + ca-certificates \ + dash \ + tzdata \ + git \ + build-essential \ + cmake \ + curl \ + libcurlpp-dev \ + zlib1g-dev \ + nano \ + tmux \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists + +# Install the RTI Connext Debian Package +RUN curl -sSL -o /usr/share/keyrings/rti-official-archive.gpg \ + https://packages.rti.com/deb/official/repo.key + +RUN printf -- "deb [arch=%s, signed-by=%s] %s %s main\n" \ + $(dpkg --print-architecture) \ + /usr/share/keyrings/rti-official-archive.gpg \ + https://packages.rti.com/deb/official \ + $(. /etc/os-release && echo ${VERSION_CODENAME}) | tee /etc/apt/sources.list.d/rti-official.list >/dev/null + +RUN export DEBIAN_FRONTEND=noninteractive \ + RTI_LICENSE_AGREEMENT_ACCEPTED=${RTI_LICENSE_AGREEMENT_ACCEPTED} \ + && apt-get update \ + && apt-get install -y \ + rti-connext-dds-${CONNEXT_VERSION}-services-routing \ + rti-connext-dds-${CONNEXT_VERSION}-services-routing-dev \ + rti-connext-dds-${CONNEXT_VERSION}-lib-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists + +WORKDIR /root + +# Copy the source code +RUN mkdir -p /root/opentelemetry-adapter/generator /root/opentelemetry-adapter/resources /root/opentelemetry-adapter/src +COPY generator /root/opentelemetry-adapter/generator +COPY resource /root/opentelemetry-adapter/resource +COPY src /root/opentelemetry-adapter/src +COPY CMakeLists.txt /root/opentelemetry-adapter/ + +# Clone the opentelemetry-cpp v1.18.0 branch without history and build +RUN git clone --depth 1 --branch v1.18.0 https://github.com/open-telemetry/opentelemetry-cpp.git +WORKDIR /root/opentelemetry-cpp +RUN git submodule update --init --recursive third_party/prometheus-cpp +RUN mkdir -p /root/opentelemetry-cpp/build +WORKDIR /root/opentelemetry-cpp/build +# Build with ABI version 2 to permit gauge metrics +RUN cmake -DBUILD_TESTING=OFF -DWITH_PROMETHEUS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DWITH_ABI_VERSION_1=OFF .. +RUN cmake --build . +RUN cmake --install . --prefix /root/opentelemetry-adapter/otel-cpp + +WORKDIR /root/opentelemetry-adapter + +# Build the source code +RUN mkdir build && cd build && cmake .. && cmake --build . + +# Configure licence file +RUN echo "export RTI_LICENSE_FILE=/root/rti_license.dat" >> /root/.bashrc +RUN rm ${NDDSHOME}/rti_license.dat +RUN ln -s /root/rti_licence.dat ${NDDSHOME}/rti_license.dat + +# Tweaks +RUN echo "source ${NDDSHOME}/resource/scripts/rtisetenv_*.bash" >> /root/.bashrc +RUN echo 'mkcd() { mkdir -p "$1" && cd "$1"; }' >> /root/.bashrc + +FROM scratch AS final-stage + +WORKDIR /root + +COPY --from=install-stage / / + +CMD ["/bin/bash"] diff --git a/AppTelemetry/LICENCE b/AppTelemetry/LICENCE new file mode 100644 index 00000000..12b9efb4 --- /dev/null +++ b/AppTelemetry/LICENCE @@ -0,0 +1,3 @@ +(c) 2005-2025 Copyright, Real-Time Innovations, Inc. (RTI) All rights reserved. + +RTI grants Licensee a license to use, modify, compile, and create derivative works of the software solely for use with RTI Connext DDS. Licensee may redistribute copies of the software, provided that all such copies are subject to this license. The software is provided "as is", with no warranty of any type, including any warranty for fitness for any purpose. RTI is under no obligation to maintain or support the software. RTI shall not be liable for any incidental or consequential damages arising out of the use or inability to use the software. \ No newline at end of file diff --git a/AppTelemetry/README.md b/AppTelemetry/README.md new file mode 100644 index 00000000..81b407e9 --- /dev/null +++ b/AppTelemetry/README.md @@ -0,0 +1,418 @@ +![Image](https://www.rti.com/hubfs/RTI_Oct2016/Images/rti-logounit.png) + +RTI Connext DDS Use Case: +Including Application Telemetry alongside RTI's Observability Platform +======================================================================= + +## Introduction + +Application telemetry refers to the collection, monitoring, and analysis of data generated by an application during its operation. This data provides critical insights into an application's performance, usage patterns, and potential issues, enabling developers and operators to optimize functionality, diagnose problems, and ensure reliability. Telemetry is essential for understanding the health and efficiency of modern distributed systems, where real-time visibility into application-level behavior is a cornerstone of successful system management. + +Telemetry data can be generated at three different levels: +* Application: Telemetry data generated when you instrument your own applications. +* Middleware: Telemetry data generated by Connext DDS entities and infrastructure services. +* System: DevOps telemetry such as CPU, memory, and disk I/O usage. + +[RTI® Connext® Observability Framework](https://community.rti.com/static/documentation/connext-dds/current/doc/manuals/addon_products/observability/index.html) is a holistic solution that uses telemetry data to provide deep visibility into the current and past states of your Connext applications. This visibility makes it easier to proactively identify and resolve potential system issues, providing a higher level of confidence in the reliable operation of the system. + +At the time of writing, RTI's Observability Framework supports middleware telemetry (metrics and logs) and application logs only. While visibility into the middleware is of great value, it would be beneficial to be able to capture and include application-level metrics in the dashboards. + +This example details the process of creating an adapter for [RTI's Routing Service](https://community.rti.com/static/documentation/connext-dds/current/doc/manuals/connext_dds_professional/services/routing_service/index.html) to collect application-level metrics published on a DDS topic. The adapter will transform and expose this telemetry data in a format compatible with Prometheus, a widely-used open-source monitoring system. Additionally you will learn how to visualize the application-level metrics alongside Connext DDS telemetry in a Grafana dashboard, enabling a unified and comprehensive monitoring experience. + +Through this guide, you will learn to: +1. Develop a custom Routing Service adapter for application telemetry. +2. Configure Prometheus to scrape and display application telemetry data. +3. Integrate application telemetry into Grafana for a combined view with Connext telemetry. + +By the end, you'll have a robust solution for monitoring both DDS system performance and application-level insights, helping you maximise the value of your telemetry data. + + +## Motivation +Why might it be preferable to use RTI's Routing Service with an OpenTelemetry adapter to collect telemetry data for Prometheus, rather than directly using the OpenTelemetry SDK or some other observability APIs? + +This method offers several advantages over directly using the **OpenTelemetry SDK** or other observability APIs: + +- **Centralized Data Collection**: Routing Service allows the collection of telemetry from multiple distributed DDS applications. +- **Efficient Data Processing and Filtering**: The Routing Service adapter can preprocess telemetry data, i.e. **content filtering, aggregation, or transformation**, before offering it to Prometheus, reducing unnecessary traffic and making telemetry collection more efficient. +- **Built-in Scalability**: By offloading telemetry collection and processing to Routing Service, **DDS applications remain focused on their primary functions**, avoiding performance degradation. Applications can be added, and additionally new Metrics can be added to applications with no changes required to the adapter. +- **Easier Deployment and Maintenance**: The OpenTelemetry adapter can be configured within Routing Service without modifying application code. +- **Better Observability in Complex DDS Architectures**: Routing Service can **correlate telemetry data across different DDS domains, partitions, and topics**, providing a more **comprehensive view** of the system. + +## Cloning the repository + +To clone the repository you will need to run git clone as follows to download both the repository and its submodule dependencies: + +To clone the [GitHub repository](https://github.com/rticommunity/rticonnextdds-usecases.git) containing this use case, you will need to run git clone as follows to download both the repository and its submodule dependencies: + +```bash +git clone --recurse-submodule +https://github.com/rticommunity/rticonnextdds-usecases.git +``` +If you forget to clone the repository with --recurse-submodule, simply run the following command to pull all the dependencies: +```bash +git submodule update --init --recursive +``` + +## Repository layout +The respository is organised as follows: + +```bash +├── CMakeLists.txt # This is the CMake file for OpenTelemetry C++ (downloaded by dockerfile), the Routing Service adapter and the generator +├── Dockerfile # The Dockerfile which tells docker how to build the container image +├── generator # The source code for the generator project, including the datamodel idl and the QoS file +│   ├── c++11 +│   │   ├── application.hpp +│   │   ├── generator.cxx +│   │   └── generator_qos.xml +│   └── telemetry.idl +├── LICENCE # The licence for the repository +├── README.md # This file +├── resource +│   ├── cmake # Additional resources required for cmake +│   └── img # Image file used in this document +└── src # The source code for the Routing Service adapter, requires OpenTelemetry C++ to compile + ├── oteladapter.cxx + ├── oteladapter.hpp + ├── otelconfig.hpp + ├── otelconnection.cxx + ├── otelconnection.hpp + ├── otelstreamwriter.cxx + ├── otelstreamwriter.hpp + ├── RsTelemetryGateway.xml # The Gateway configuration + └── run_tmux.sh # Helper script to run both applications simultaneously in a single docker terminal window +``` + + + +## Build Docker image + +This example is encapsulated inside a Docker container for convenience, and once the repository has been cloned, the next step is to configure a Docker image. During the image build process, the applications included in this example will be automatically compiled and included inside the Docker image. These applications and the sources will be accessible in the resulting image at the /root/ directory. This results in a sandbox which allows you to explore, modify and build the code as desired without tainting the host system with library sources and build artefacts. + +To create the docker image adjust the line in the Dockerfile that reads ENV TZ=Europe/Madrid to set the correct timezone, then run the following command to build the Connext image for the 7.3.0 release, take into account that you are accepting the RTI license agreement by setting the RTI_LICENSE_AGREEMENT_ACCEPTED argument to "accepted": + +```bash +docker build -t connext:oteladapter -f ./Dockerfile --build-arg RTI_LICENSE_AGREEMENT_ACCEPTED=accepted --build-arg CONNEXT_VERSION=7.3.0 . +``` + +If you want to target a different version of Connext, then you must set the following arguments to match your requirements +- CONNEXT_VERSION: supported values are 7.3.0 and 6.1.2 + +Here's an example of what that would look like for a 6.1.2 release: +```bash +docker build -t connext:6.1.2 -f docker/Dockerfile --build-arg RTI_LICENSE_AGREEMENT_ACCEPTED=accepted --build-arg CONNEXT_VERSION=6.1.2 . +``` + +# Running the example +## Starting the Docker Container + To run the Docker container, run the following command: + +```bash +docker run -it --rm --net=host -v $RTI_LICENSE_FILE:/root/rti_license.dat:ro connext:oteladapter +``` + +### Here’s what those parameters mean: +- run: run container +- -i: Interactive +- -t: Allocate pseudo-TTY +- --rm: Automatically remove the container when it exits +- –net: Define which docker network to use, in this case the container shares its network namespace with the host machine +- -v: Bind mount a volume, this line ensures that the local Connext license appears inside the Docker container as a read-only file + + +A valid RTI Connext license is required to execute Connext within the containers, you can download a fully functional Connext evaluation license [here](https://www.rti.com/free-trial). To set the required environment variable `RTI_LICENSE_FILE`, in the terminal from which you start the docker containers, run: +```bash +export RTI_LICENSE_FILE=/path/to/rti_license.dat +``` + +## Executing the example +Once the container is started and you are at the terminal prompt, change directory to the `opentelemetry-adapter/build` directory. From there you can execute the run_tmux.sh script which will create a tmux session, split the window two panes and starts the two parts of the demo: + +![tmux session](resource/img/tmux.png) + +The Prometheus Gateway is in the top pane, and the Generator can be seen running in the lower pane. You can switch between the two panes to control them. +### Switching Panes in tmux + +To switch between panes in a [tmux](https://github.com/tmux/tmux/wiki/Getting-Started) session, you can use the following keyboard shortcuts: + +- `Ctrl-b` followed by `o`: Move to the next pane. + +- `Ctrl-b` followed by an arrow key (`←`, `→`, `↑`, `↓`): Move to the pane in the specified direction. + +- `Ctrl-b` followed by `;`: Move to the previously active pane. + +These shortcuts allow you to navigate efficiently between different panes within your tmux session. + +# How the example was developed + +## Step 1: Define the data model + +The first challenge is to come up with [a data model](generator/telemetry.idl) that reflects our needs: routing telemetry data to Prometheus via DDS. The most straight-forward way would be to establish a data model which defines a set of metrics and their associated data structures which can be easily mapped to OpenTelemetry concepts. The use of `@appendable` annotations ensures extensibility, allowing the model to evolve without breaking compatibility, which is crucial for observability frameworks like OpenTelemetry. + +### Main structure +There are various types of metric that we might want to support along with descriptive data. So let's define that first: + +```c +@appendable +struct Metric { + @key MetricName name; + MetricDescription description; + @key MetricUnit unit; + MetricUnion data; + @optional sequence