Bring in VideoFrame FrameMetadata from FFI

Author: stephen-derosa

examples/CMakeLists.txt

@@ -39,6 +39,8 @@ set(EXAMPLES_COMMON_DIR ${CMAKE_CURRENT_SOURCE_DIR}/common)
 # All example executables (used for copying livekit_ffi DLL/shared lib)
 set(EXAMPLES_ALL
   SimpleRoom
+  UserTimestampedVideoProducer
+  UserTimestampedVideoConsumer
   SimpleRpc
   SimpleJoystickSender
   SimpleJoystickReceiver
@@ -234,6 +236,32 @@ target_link_libraries(SimpleDataStream
     spdlog::spdlog
 )
 
+# --- UserTimestampedVideo example ---
+
+add_executable(UserTimestampedVideoProducer
+  user_timestamped_video/producer.cpp
+)
+
+target_include_directories(UserTimestampedVideoProducer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS})
+
+target_link_libraries(UserTimestampedVideoProducer
+  PRIVATE
+    livekit
+    spdlog::spdlog
+)
+
+add_executable(UserTimestampedVideoConsumer
+  user_timestamped_video/consumer.cpp
+)
+
+target_include_directories(UserTimestampedVideoConsumer PRIVATE ${EXAMPLES_PRIVATE_INCLUDE_DIRS})
+
+target_link_libraries(UserTimestampedVideoConsumer
+  PRIVATE
+    livekit
+    spdlog::spdlog
+)
+
 add_custom_command(
   TARGET SimpleDataStream
   POST_BUILD

examples/common/sdl_media_manager.cpp

@@ -399,4 +399,4 @@ void SDLMediaManager::render() {
   if (renderer_running_.load(std::memory_order_relaxed) && sdl_renderer_) {
     sdl_renderer_->render();
   }
-}
+}

examples/user_timestamped_video/README.md

@@ -0,0 +1,53 @@
+# UserTimestampedVideo
+
+This example is split into two executables and can demonstrate all four
+producer/consumer combinations:
+
+- `UserTimestampedVideoProducer` publishes a synthetic camera track and stamps
+  each frame with `VideoCaptureOptions::metadata.user_timestamp_us`.
+- `UserTimestampedVideoConsumer` subscribes to remote camera frames with
+  either the rich or legacy callback path.
+
+Run them in the same room with different participant identities:
+
+```sh
+LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN=<producer-token> ./UserTimestampedVideoProducer
+LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN=<consumer-token> ./UserTimestampedVideoConsumer
+```
+
+Flags:
+
+- Producer default: sends user timestamps
+- Producer `--without-user-timestamp`: does not send user timestamps
+- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback`
+- Consumer `--ignore-user-timestamp`: ignores metadata through the legacy
+  `setOnVideoFrameCallback`
+
+Matrix:
+
+```sh
+# 1. Producer sends, consumer reads
+./UserTimestampedVideoProducer
+./UserTimestampedVideoConsumer
+
+# 2. Producer sends, consumer ignores
+./UserTimestampedVideoProducer
+./UserTimestampedVideoConsumer --ignore-user-timestamp
+
+# 3. Producer does not send, consumer ignores
+./UserTimestampedVideoProducer --without-user-timestamp
+./UserTimestampedVideoConsumer --ignore-user-timestamp
+
+# 4. Producer does not send, consumer reads
+./UserTimestampedVideoProducer --without-user-timestamp
+./UserTimestampedVideoConsumer
+```
+
+Timestamp note:
+
+- `user_ts_us` is application metadata and is the value to compare end to end.
+- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`.
+- `capture_ts_us` on the consumer is the received WebRTC frame timestamp.
+- Producer and consumer `capture_ts_us` values are not expected to match exactly,
+  because WebRTC may translate frame timestamps onto its own internal
+  capture-time timeline before delivery.

examples/user_timestamped_video/consumer.cpp

@@ -0,0 +1,247 @@
+/*
+ * Copyright 2026 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * UserTimestampedVideoConsumer
+ *
+ * Receives remote camera frames via Room::setOnVideoFrameEventCallback() and
+ * logs any VideoFrameMetadata::user_timestamp_us values that arrive.
+ */
+
+#include <atomic>
+#include <csignal>
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <thread>
+#include <unordered_set>
+#include <vector>
+
+#include "livekit/livekit.h"
+
+using namespace livekit;
+
+namespace {
+
+std::atomic<bool> g_running{true};
+
+void handleSignal(int) { g_running.store(false); }
+
+std::string getenvOrEmpty(const char *name) {
+  const char *value = std::getenv(name);
+  return value ? std::string(value) : std::string{};
+}
+
+std::string
+formatUserTimestamp(const std::optional<VideoFrameMetadata> &metadata) {
+  if (!metadata || !metadata->user_timestamp_us.has_value()) {
+    return "n/a";
+  }
+
+  return std::to_string(*metadata->user_timestamp_us);
+}
+
+void printUsage(const char *program) {
+  std::cerr << "Usage:\n"
+            << "  " << program << " <ws-url> <token> [--ignore-user-timestamp]\n"
+            << "or:\n"
+            << "  LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program
+            << " [--ignore-user-timestamp]\n";
+}
+
+bool parseArgs(int argc, char *argv[], std::string &url, std::string &token,
+               bool &read_user_timestamp) {
+  read_user_timestamp = true;
+  std::vector<std::string> positional;
+
+  for (int i = 1; i < argc; ++i) {
+    const std::string arg = argv[i];
+    if (arg == "-h" || arg == "--help") {
+      return false;
+    }
+    if (arg == "--ignore-user-timestamp") {
+      read_user_timestamp = false;
+      continue;
+    }
+    if (arg == "--read-user-timestamp") {
+      read_user_timestamp = true;
+      continue;
+    }
+
+    positional.push_back(arg);
+  }
+
+  url = getenvOrEmpty("LIVEKIT_URL");
+  token = getenvOrEmpty("LIVEKIT_TOKEN");
+
+  if (positional.size() >= 2) {
+    url = positional[0];
+    token = positional[1];
+  }
+
+  return !(url.empty() || token.empty());
+}
+
+class UserTimestampedVideoConsumerDelegate : public RoomDelegate {
+public:
+  UserTimestampedVideoConsumerDelegate(Room &room, bool read_user_timestamp)
+      : room_(room), read_user_timestamp_(read_user_timestamp) {}
+
+  void registerExistingParticipants() {
+    for (const auto &participant : room_.remoteParticipants()) {
+      if (participant) {
+        registerRemoteCameraCallback(participant->identity());
+      }
+    }
+  }
+
+  void onParticipantConnected(Room &,
+                              const ParticipantConnectedEvent &event) override {
+    if (!event.participant) {
+      return;
+    }
+
+    std::cout << "[consumer] participant connected: "
+              << event.participant->identity() << "\n";
+    registerRemoteCameraCallback(event.participant->identity());
+  }
+
+  void onParticipantDisconnected(
+      Room &, const ParticipantDisconnectedEvent &event) override {
+    if (!event.participant) {
+      return;
+    }
+
+    const std::string identity = event.participant->identity();
+    room_.clearOnVideoFrameCallback(identity, TrackSource::SOURCE_CAMERA);
+
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      registered_identities_.erase(identity);
+    }
+
+    std::cout << "[consumer] participant disconnected: " << identity << "\n";
+  }
+
+private:
+  void registerRemoteCameraCallback(const std::string &identity) {
+    {
+      std::lock_guard<std::mutex> lock(mutex_);
+      if (!registered_identities_.insert(identity).second) {
+        return;
+      }
+    }
+
+    VideoStream::Options stream_options;
+    stream_options.format = VideoBufferType::RGBA;
+
+    if (read_user_timestamp_) {
+      room_.setOnVideoFrameEventCallback(
+          identity, TrackSource::SOURCE_CAMERA,
+          [identity](const VideoFrameEvent &event) {
+            std::cout << "[consumer] from=" << identity
+                      << " size=" << event.frame.width() << "x"
+                      << event.frame.height()
+                      << " capture_ts_us=" << event.timestamp_us
+                      << " user_ts_us=" << formatUserTimestamp(event.metadata)
+                      << " rotation=" << static_cast<int>(event.rotation)
+                      << "\n";
+          },
+          stream_options);
+    } else {
+      room_.setOnVideoFrameCallback(
+          identity, TrackSource::SOURCE_CAMERA,
+          [identity](const VideoFrame &frame, const std::int64_t timestamp_us) {
+            std::cout << "[consumer] from=" << identity
+                      << " size=" << frame.width() << "x" << frame.height()
+                      << " capture_ts_us=" << timestamp_us
+                      << " user_ts_us=ignored\n";
+          },
+          stream_options);
+    }
+
+    std::cout << "[consumer] listening for camera frames from " << identity
+              << " with user timestamp "
+              << (read_user_timestamp_ ? "enabled" : "ignored") << "\n";
+  }
+
+  Room &room_;
+  bool read_user_timestamp_;
+  std::mutex mutex_;
+  std::unordered_set<std::string> registered_identities_;
+};
+
+} // namespace
+
+int main(int argc, char *argv[]) {
+  std::string url;
+  std::string token;
+  bool read_user_timestamp = true;
+
+  if (!parseArgs(argc, argv, url, token, read_user_timestamp)) {
+    printUsage(argv[0]);
+    return 1;
+  }
+
+  std::signal(SIGINT, handleSignal);
+#ifdef SIGTERM
+  std::signal(SIGTERM, handleSignal);
+#endif
+
+  livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole);
+  int exit_code = 0;
+
+  {
+    Room room;
+    RoomOptions options;
+    options.auto_subscribe = true;
+    options.dynacast = false;
+
+    UserTimestampedVideoConsumerDelegate delegate(room, read_user_timestamp);
+    room.setDelegate(&delegate);
+
+    std::cout << "[consumer] connecting to " << url << "\n";
+    if (!room.Connect(url, token, options)) {
+      std::cerr << "[consumer] failed to connect\n";
+      exit_code = 1;
+    } else {
+      std::cout << "[consumer] connected as "
+                << room.localParticipant()->identity() << " to room '"
+                << room.room_info().name << "' with user timestamp "
+                << (read_user_timestamp ? "enabled" : "ignored") << "\n";
+
+      delegate.registerExistingParticipants();
+
+      while (g_running.load(std::memory_order_relaxed)) {
+        std::this_thread::sleep_for(std::chrono::milliseconds(50));
+      }
+
+      for (const auto &participant : room.remoteParticipants()) {
+        if (participant) {
+          room.clearOnVideoFrameCallback(participant->identity(),
+                                         TrackSource::SOURCE_CAMERA);
+        }
+      }
+    }
+
+    room.setDelegate(nullptr);
+  }
+
+  livekit::shutdown();
+  return exit_code;
+}