From cc552853624c56c2de633839cfdf05300a791af1 Mon Sep 17 00:00:00 2001
From: Artur <35294812+arturoptophys@users.noreply.github.com>
Date: Tue, 21 Oct 2025 11:22:49 +0200
Subject: [PATCH 01/26] Add modular camera backends with Basler and GenTL
support
---
.gitignore | 2 +
README.md | 165 +-
dlclivegui/__init__.py | 21 +-
dlclivegui/camera/__init__.py | 43 -
dlclivegui/camera/aravis.py | 128 --
dlclivegui/camera/basler.py | 104 --
dlclivegui/camera/camera.py | 139 --
dlclivegui/camera/opencv.py | 155 --
dlclivegui/camera/pseye.py | 101 --
dlclivegui/camera/tiscamera_linux.py | 204 ---
dlclivegui/camera/tiscamera_windows.py | 129 --
dlclivegui/camera/tisgrabber_windows.py | 781 ---------
dlclivegui/camera_controller.py | 120 ++
dlclivegui/camera_process.py | 338 ----
dlclivegui/cameras/__init__.py | 6 +
dlclivegui/cameras/base.py | 46 +
dlclivegui/cameras/basler_backend.py | 132 ++
dlclivegui/cameras/factory.py | 70 +
dlclivegui/cameras/gentl_backend.py | 130 ++
dlclivegui/cameras/opencv_backend.py | 61 +
dlclivegui/config.py | 112 ++
dlclivegui/dlc_processor.py | 123 ++
dlclivegui/dlclivegui.py | 1498 -----------------
dlclivegui/gui.py | 542 ++++++
dlclivegui/pose_process.py | 273 ---
dlclivegui/processor/__init__.py | 1 -
dlclivegui/processor/processor.py | 23 -
dlclivegui/processor/teensy_laser/__init__.py | 1 -
.../processor/teensy_laser/teensy_laser.ino | 77 -
.../processor/teensy_laser/teensy_laser.py | 77 -
dlclivegui/queue.py | 208 ---
dlclivegui/tkutil.py | 195 ---
dlclivegui/video.py | 274 ---
dlclivegui/video_recorder.py | 46 +
setup.py | 35 +-
35 files changed, 1533 insertions(+), 4827 deletions(-)
delete mode 100644 dlclivegui/camera/__init__.py
delete mode 100644 dlclivegui/camera/aravis.py
delete mode 100644 dlclivegui/camera/basler.py
delete mode 100644 dlclivegui/camera/camera.py
delete mode 100644 dlclivegui/camera/opencv.py
delete mode 100644 dlclivegui/camera/pseye.py
delete mode 100644 dlclivegui/camera/tiscamera_linux.py
delete mode 100644 dlclivegui/camera/tiscamera_windows.py
delete mode 100644 dlclivegui/camera/tisgrabber_windows.py
create mode 100644 dlclivegui/camera_controller.py
delete mode 100644 dlclivegui/camera_process.py
create mode 100644 dlclivegui/cameras/__init__.py
create mode 100644 dlclivegui/cameras/base.py
create mode 100644 dlclivegui/cameras/basler_backend.py
create mode 100644 dlclivegui/cameras/factory.py
create mode 100644 dlclivegui/cameras/gentl_backend.py
create mode 100644 dlclivegui/cameras/opencv_backend.py
create mode 100644 dlclivegui/config.py
create mode 100644 dlclivegui/dlc_processor.py
delete mode 100644 dlclivegui/dlclivegui.py
create mode 100644 dlclivegui/gui.py
delete mode 100644 dlclivegui/pose_process.py
delete mode 100644 dlclivegui/processor/__init__.py
delete mode 100644 dlclivegui/processor/processor.py
delete mode 100644 dlclivegui/processor/teensy_laser/__init__.py
delete mode 100644 dlclivegui/processor/teensy_laser/teensy_laser.ino
delete mode 100644 dlclivegui/processor/teensy_laser/teensy_laser.py
delete mode 100644 dlclivegui/queue.py
delete mode 100644 dlclivegui/tkutil.py
delete mode 100644 dlclivegui/video.py
create mode 100644 dlclivegui/video_recorder.py
diff --git a/.gitignore b/.gitignore
index 208ed2c..1a13ced 100644
--- a/.gitignore
+++ b/.gitignore
@@ -116,3 +116,5 @@ venv.bak/
# ide files
.vscode
+
+!dlclivegui/config.py
diff --git a/README.md b/README.md
index acdadf2..a886a98 100644
--- a/README.md
+++ b/README.md
@@ -1,69 +1,126 @@
-# DeepLabCut-Live! GUI 
-
-
-
-
-
-
-[](https://github.com/DeepLabCut/deeplabcutlive/raw/master/LICENSE)
-[](https://forum.image.sc/tags/deeplabcut)
-[](https://gitter.im/DeepLabCut/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
-[](https://twitter.com/DeepLabCut)
-
-GUI to run [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) on a video feed, record videos, and record external timestamps.
-
-## [Installation Instructions](docs/install.md)
-
-## Getting Started
-
-#### Open DeepLabCut-live-GUI
-
-In a terminal, activate the conda or virtual environment where DeepLabCut-live-GUI is installed, then run:
-
-```
-dlclivegui
+# DeepLabCut Live GUI
+
+A modernised PyQt6 GUI for running [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) experiments. The application
+streams frames from a camera, optionally performs DLCLive inference, and records video using the
+[vidgear](https://github.com/abhiTronix/vidgear) toolkit.
+
+## Features
+
+- Python 3.11+ compatible codebase with a PyQt6 interface.
+- Modular architecture with dedicated modules for camera control, video recording, configuration
+ management, and DLCLive processing.
+- Single JSON configuration file that captures camera settings, DLCLive parameters, and recording
+ options. All fields can be edited directly within the GUI.
+- Optional DLCLive inference with pose visualisation over the live video feed.
+- Recording support via vidgear's `WriteGear`, including custom encoder options.
+
+## Installation
+
+1. Install the package and its dependencies:
+
+ ```bash
+ pip install deeplabcut-live-gui
+ ```
+
+ The GUI requires additional runtime packages for optional features:
+
+ - [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) for pose estimation.
+ - [vidgear](https://github.com/abhiTronix/vidgear) for video recording.
+ - [OpenCV](https://opencv.org/) for camera access.
+
+ These libraries are listed in `setup.py` and will be installed automatically when the package is
+ installed via `pip`.
+
+2. Launch the GUI:
+
+ ```bash
+ dlclivegui
+ ```
+
+## Configuration
+
+The GUI works with a single JSON configuration describing the experiment. The configuration contains
+three main sections:
+
+```json
+{
+ "camera": {
+ "index": 0,
+ "width": 1280,
+ "height": 720,
+ "fps": 60.0,
+ "backend": "opencv",
+ "properties": {}
+ },
+ "dlc": {
+ "model_path": "/path/to/exported-model",
+ "processor": "cpu",
+ "shuffle": 1,
+ "trainingsetindex": 0,
+ "processor_args": {},
+ "additional_options": {}
+ },
+ "recording": {
+ "enabled": true,
+ "directory": "~/Videos/deeplabcut",
+ "filename": "session.mp4",
+ "container": "mp4",
+ "options": {
+ "compression_mode": "mp4"
+ }
+ }
+}
```
+Use **File → Load configuration…** to open an existing configuration, or **File → Save configuration**
+to persist the current settings. Every field in the GUI is editable, and values entered in the
+interface will be written back to the JSON file.
-#### Configurations
+### Camera backends
+Set `camera.backend` to one of the supported drivers:
-First, create a configuration file: select the drop down menu labeled `Config`, and click `Create New Config`. All settings, such as details about cameras, DLC networks, and DLC-live Processors, will be saved into configuration files so that you can close and reopen the GUI without losing all of these details. You can create multiple configuration files on the same system, so that different users can save different camera options, etc on the same computer. To load previous settings from a configuration file, please just select the file from the drop-down menu. Configuration files are stored at `$HOME/Documents/DeepLabCut-live-GUI/config`. These files do not need to be edited manually, they can be entirely created and edited automatically within the GUI.
+- `opencv` – standard `cv2.VideoCapture` fallback available on every platform.
+- `basler` – uses the Basler Pylon SDK via `pypylon` (install separately).
+- `gentl` – uses Aravis for GenTL-compatible cameras (requires `python-gi` bindings).
-#### Set Up Cameras 
+Backend specific parameters can be supplied through the `camera.properties` object. For example:
-To setup a new camera, select `Add Camera` from the dropdown menu, and then click `Init Cam`. This will be bring up a new window where you need to select the type of camera (see [Camera Support](docs/camera_support.md)), input a name for the camera, and click `Add Camera`. This will initialize a new `Camera` entry in the drop down menu. Now, select your camera from the dropdown menu and click`Edit Camera Settings` to setup your camera settings (i.e. set the serial number, exposure, cropping parameters, etc; the exact settings depend on the specific type of camera). Once you have set the camera settings, click `Init Cam` to start streaming. To stop streaming data, click `Close Camera`, and to remove a camera from the dropdown menu, click `Remove Camera`.
-
-#### Processor (optional)
-
-To write custom `Processors`, please see [here](https://github.com/DeepLabCut/DeepLabCut-live/tree/master/dlclive/processor). The directory that contains your custom `Processor` should be a python module -- this directory must contain an `__init__.py` file that imports your custom `Processor`. For examples of how to structure a custom `Processor` directory, please see [here](https://github.com/DeepLabCut/DeepLabCut-live/tree/master/example_processors).
-
-To use your processor in the GUI, you must first add your custom `Processor` directory to the dropdown menu: next to the `Processor Dir` label, click `Browse`, and select your custom `Processor` directory. Next, select the desired directory from the `Processor Dir` dropdown menu, then select the `Processor` you would like to use from the `Processor` menu. If you would like to edit the arguments for your processor, please select `Edit Proc Settings`, and finally, to use the processor, click `Set Proc`. If you have previously set a `Processor` and would like to clear it, click `Clear Proc`.
-
-#### Configure DeepLabCut Network
-
-
-
-Select the `DeepLabCut` dropdown menu, and click `Add DLC`. This will bring up a new window to choose a name for the DeepLabCut configuration, choose the path to the exported DeepLabCut model, and set DeepLabCut-live settings, such as the cropping or resize parameters. Once configured, click `Update` to add this DeepLabCut configuration to the dropdown menu. You can edit the settings at any time by clicking `Edit DLC Settings`. Once configured, you can load the network and start performing inference by clicking `Start DLC`. If you would like to view the DeepLabCut pose estimation in real-time, select `Display DLC Keypoints`. You can edit the keypoint display settings (the color scheme, size of points, and the likelihood threshold for display) by selecting `Edit DLC Display Settings`.
-
-If you want to stop performing inference at any time, just click `Stop DLC`, and if you want to remove a DeepLabCut configuration from the dropdown menu, click `Remove DLC`.
+```json
+{
+ "camera": {
+ "index": 0,
+ "backend": "basler",
+ "properties": {
+ "serial": "40123456",
+ "exposure": 15000,
+ "gain": 6.0
+ }
+ }
+}
+```
-#### Set Up Session
+If optional dependencies are missing, the GUI will show the backend as unavailable in the drop-down
+but you can still configure it for a system where the drivers are present.
-Sessions are defined by the subject name, the date, and an attempt number. Within the GUI, select a `Subject` from the dropdown menu, or to add a new subject, type the new subject name in to the entry box and click `Add Subject`. Next, select an `Attempt` from the dropdown menu. Then, select the directory that you would like to save data to from the `Directory` dropdown menu. To add a new directory to the dropdown menu, click `Browse`. Finally, click `Set Up Session` to initiate a new recording. This will prepare the GUI to save data. Once you click `Set Up Session`, the `Ready` button should turn blue, indicating a session is ready to record.
+## Development
-#### Controlling Recording
+The core modules of the package are organised as follows:
-If the `Ready` button is selected, you can now start a recording by clicking `On`. The `On` button will then turn green indicating a recording is active. To stop a recording, click `Off`. This will cause the `Ready` button to be selected again, as the GUI is prepared to restart the recording and to save the data to the same file. If you're session is complete, click `Save Video` to save all files: the video recording (as .avi file), a numpy file with timestamps for each recorded frame, the DeepLabCut poses as a pandas data frame (hdf5 file) that includes the time of each frame used for pose estimation and the time that each pose was obtained, and if applicable, files saved by the `Processor` in use. These files will be saved into a new directory at `{YOUR_SAVE_DIRECTORY}/{CAMERA NAME}_{SUBJECT}_{DATE}_{ATTEMPT}`
+- `dlclivegui.config` – dataclasses for loading, storing, and saving application settings.
+- `dlclivegui.cameras` – modular camera backends (OpenCV, Basler, GenTL) and factory helpers.
+- `dlclivegui.camera_controller` – camera capture worker running in a dedicated `QThread`.
+- `dlclivegui.video_recorder` – wrapper around `WriteGear` for video output.
+- `dlclivegui.dlc_processor` – asynchronous DLCLive inference with optional pose overlay.
+- `dlclivegui.gui` – PyQt6 user interface and application entry point.
-- YOUR_SAVE_DIRECTORY : the directory chosen from the `Directory` dropdown menu.
-- CAMERA NAME : the name of selected camera (from the `Camera` dropdown menu).
-- SUBJECT : the subject chosen from the `Subject` drowdown menu.
-- DATE : the current date of the experiment.
-- ATTEMPT : the attempt number chosen from the `Attempt` dropdown.
+Run a quick syntax check with:
-If you would not like to save the data from the session, please click `Delete Video`, and all data will be discarded. After you click `Save Video` or `Delete Video`, the `Off` button will be selected, indicating you can now set up a new session.
+```bash
+python -m compileall dlclivegui
+```
-#### References:
+## License
-If you use this code we kindly ask you to you please [cite Kane et al, eLife 2020](https://elifesciences.org/articles/61909). The preprint is available here: https://www.biorxiv.org/content/10.1101/2020.08.04.236422v2
+This project is licensed under the GNU Lesser General Public License v3.0. See the `LICENSE` file for
+more information.
diff --git a/dlclivegui/__init__.py b/dlclivegui/__init__.py
index 1583156..1408486 100644
--- a/dlclivegui/__init__.py
+++ b/dlclivegui/__init__.py
@@ -1,4 +1,17 @@
-from dlclivegui.camera_process import CameraProcess
-from dlclivegui.pose_process import CameraPoseProcess
-from dlclivegui.video import create_labeled_video
-from dlclivegui.dlclivegui import DLCLiveGUI
+"""DeepLabCut Live GUI package."""
+from .config import (
+ ApplicationSettings,
+ CameraSettings,
+ DLCProcessorSettings,
+ RecordingSettings,
+)
+from .gui import MainWindow, main
+
+__all__ = [
+ "ApplicationSettings",
+ "CameraSettings",
+ "DLCProcessorSettings",
+ "RecordingSettings",
+ "MainWindow",
+ "main",
+]
diff --git a/dlclivegui/camera/__init__.py b/dlclivegui/camera/__init__.py
deleted file mode 100644
index 2368198..0000000
--- a/dlclivegui/camera/__init__.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import platform
-
-from dlclivegui.camera.camera import Camera, CameraError
-from dlclivegui.camera.opencv import OpenCVCam
-
-if platform.system() == "Windows":
- try:
- from dlclivegui.camera.tiscamera_windows import TISCam
- except Exception as e:
- pass
-
-if platform.system() == "Linux":
- try:
- from dlclivegui.camera.tiscamera_linux import TISCam
- except Exception as e:
- pass
- # print(f"Error importing TISCam on Linux: {e}")
-
-if platform.system() in ["Darwin", "Linux"]:
- try:
- from dlclivegui.camera.aravis import AravisCam
- except Exception as e:
- pass
- # print(f"Error importing AravisCam: f{e}")
-
-if platform.system() == "Darwin":
- try:
- from dlclivegui.camera.pseye import PSEyeCam
- except Exception as e:
- pass
-
-try:
- from dlclivegui.camera.basler import BaslerCam
-except Exception as e:
- pass
diff --git a/dlclivegui/camera/aravis.py b/dlclivegui/camera/aravis.py
deleted file mode 100644
index 92662e1..0000000
--- a/dlclivegui/camera/aravis.py
+++ /dev/null
@@ -1,128 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import ctypes
-import numpy as np
-import time
-
-import gi
-
-gi.require_version("Aravis", "0.6")
-from gi.repository import Aravis
-import cv2
-
-from dlclivegui.camera import Camera
-
-
-class AravisCam(Camera):
- @staticmethod
- def arg_restrictions():
-
- Aravis.update_device_list()
- n_cams = Aravis.get_n_devices()
- ids = [Aravis.get_device_id(i) for i in range(n_cams)]
- return {"id": ids}
-
- def __init__(
- self,
- id="",
- resolution=[720, 540],
- exposure=0.005,
- gain=0,
- rotate=0,
- crop=None,
- fps=100,
- display=True,
- display_resize=1.0,
- ):
-
- super().__init__(
- id,
- resolution=resolution,
- exposure=exposure,
- gain=gain,
- rotate=rotate,
- crop=crop,
- fps=fps,
- use_tk_display=display,
- display_resize=display_resize,
- )
-
- def set_capture_device(self):
-
- self.cam = Aravis.Camera.new(self.id)
- self.no_auto()
- self.set_exposure(self.exposure)
- self.set_crop(self.crop)
- self.cam.set_frame_rate(self.fps)
-
- self.stream = self.cam.create_stream()
- self.stream.push_buffer(Aravis.Buffer.new_allocate(self.cam.get_payload()))
- self.cam.start_acquisition()
-
- return True
-
- def no_auto(self):
-
- self.cam.set_exposure_time_auto(0)
- self.cam.set_gain_auto(0)
-
- def set_exposure(self, val):
-
- val = 1 if val > 1 else val
- val = 0 if val < 0 else val
- self.cam.set_exposure_time(val * 1e6)
-
- def set_crop(self, crop):
-
- if crop:
- left = crop[0]
- width = crop[1] - left
- top = crop[3]
- height = top - crop[2]
- self.cam.set_region(left, top, width, height)
- self.im_size = (width, height)
-
- def get_image_on_time(self):
-
- buffer = None
- while buffer is None:
- buffer = self.stream.try_pop_buffer()
-
- frame = self._convert_image_to_numpy(buffer)
- self.stream.push_buffer(buffer)
-
- return frame, time.time()
-
- def _convert_image_to_numpy(self, buffer):
- """ from https://github.com/SintefManufacturing/python-aravis """
-
- pixel_format = buffer.get_image_pixel_format()
- bits_per_pixel = pixel_format >> 16 & 0xFF
-
- if bits_per_pixel == 8:
- INTP = ctypes.POINTER(ctypes.c_uint8)
- else:
- INTP = ctypes.POINTER(ctypes.c_uint16)
-
- addr = buffer.get_data()
- ptr = ctypes.cast(addr, INTP)
-
- frame = np.ctypeslib.as_array(
- ptr, (buffer.get_image_height(), buffer.get_image_width())
- )
- frame = frame.copy()
-
- if frame.ndim < 3:
- frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
-
- return frame
-
- def close_capture_device():
-
- self.cam.stop_acquisition()
diff --git a/dlclivegui/camera/basler.py b/dlclivegui/camera/basler.py
deleted file mode 100644
index 1706208..0000000
--- a/dlclivegui/camera/basler.py
+++ /dev/null
@@ -1,104 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-#import pypylon as pylon
-from pypylon import pylon
-from imutils import rotate_bound
-import time
-
-from dlclivegui.camera import Camera, CameraError
-TIMEOUT = 100
-
-def get_devices():
- tlFactory = pylon.TlFactory.GetInstance()
- devices = tlFactory.EnumerateDevices()
- return devices
-
-class BaslerCam(Camera):
- @staticmethod
- def arg_restrictions():
- """ Returns a dictionary of arguments restrictions for DLCLiveGUI
- """
- devices = get_devices()
- device_ids = list(range(len(devices)))
- return {"device": device_ids, "display": [True, False]}
-
- def __init__(
- self,
- device=0,
- resolution=[640, 480],
- exposure=15000,
- rotate=0,
- crop=None,
- gain=0.0,
- fps=30,
- display=True,
- display_resize=1.0,
- ):
-
- super().__init__(
- device,
- resolution=resolution,
- exposure=exposure,
- rotate=rotate,
- crop=crop,
- gain=gain,
- fps=fps,
- use_tk_display=display,
- display_resize=display_resize,
- )
-
- self.display = display
-
- def set_capture_device(self):
-
- devices = get_devices()
- self.cam = pylon.InstantCamera(
- pylon.TlFactory.GetInstance().CreateDevice(devices[self.id])
- )
- self.cam.Open()
-
- self.cam.Gain.SetValue(self.gain)
- self.cam.ExposureTime.SetValue(self.exposure)
- self.cam.Width.SetValue(self.im_size[0])
- self.cam.Height.SetValue(self.im_size[1])
-
- self.cam.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)
- self.converter = pylon.ImageFormatConverter()
- self.converter.OutputPixelFormat = pylon.PixelType_BGR8packed
- self.converter.OutputBitAlignment = pylon.OutputBitAlignment_MsbAligned
-
- return True
-
- def get_image(self):
- grabResult = self.cam.RetrieveResult(
- TIMEOUT, pylon.TimeoutHandling_ThrowException)
-
- frame = None
-
- if grabResult.GrabSucceeded():
-
- image = self.converter.Convert(grabResult)
- frame = image.GetArray()
-
- if self.rotate:
- frame = rotate_bound(frame, self.rotate)
- if self.crop:
- frame = frame[self.crop[2]: self.crop[3],
- self.crop[0]: self.crop[1]]
-
- else:
-
- raise CameraError("Basler Camera did not return an image!")
-
- grabResult.Release()
-
- return frame
-
- def close_capture_device(self):
-
- self.cam.StopGrabbing()
diff --git a/dlclivegui/camera/camera.py b/dlclivegui/camera/camera.py
deleted file mode 100644
index e81442f..0000000
--- a/dlclivegui/camera/camera.py
+++ /dev/null
@@ -1,139 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import cv2
-import time
-
-
-class CameraError(Exception):
- """
- Exception for incorrect use of cameras
- """
-
- pass
-
-
-class Camera(object):
- """ Base camera class. Controls image capture, writing images to video, pose estimation and image display.
-
- Parameters
- ----------
- id : [type]
- camera id
- exposure : int, optional
- exposure time in microseconds, by default None
- gain : int, optional
- gain value, by default None
- rotate : [type], optional
- [description], by default None
- crop : list, optional
- camera cropping parameters: [left, right, top, bottom], by default None
- fps : float, optional
- frame rate in frames per second, by default None
- use_tk_display : bool, optional
- flag to use tk image display (if using GUI), by default False
- display_resize : float, optional
- factor to resize images if using opencv display (display is very slow for large images), by default None
- """
-
- @staticmethod
- def arg_restrictions():
- """ Returns a dictionary of arguments restrictions for DLCLiveGUI
- """
-
- return {}
-
- def __init__(
- self,
- id,
- resolution=None,
- exposure=None,
- gain=None,
- rotate=None,
- crop=None,
- fps=None,
- use_tk_display=False,
- display_resize=1.0,
- ):
- """ Constructor method
- """
-
- self.id = id
- self.exposure = exposure
- self.gain = gain
- self.rotate = rotate
- self.crop = [int(c) for c in crop] if crop else None
- self.set_im_size(resolution)
- self.fps = fps
- self.use_tk_display = use_tk_display
- self.display_resize = display_resize if display_resize else 1.0
- self.next_frame = 0
-
- def set_im_size(self, res):
- """[summary]
-
- Parameters
- ----------
- default : [, optional
- [description], by default None
-
- Raises
- ------
- DLCLiveCameraError
- throws error if resolution is not set
- """
-
- if not res:
- raise CameraError("Resolution is not set!")
-
- self.im_size = (
- (int(res[0]), int(res[1]))
- if self.crop is None
- else (self.crop[3] - self.crop[2], self.crop[1] - self.crop[0])
- )
-
- def set_capture_device(self):
- """ Sets frame capture device with desired properties
- """
-
- raise NotImplementedError
-
- def get_image_on_time(self):
- """ Gets an image from frame capture device at the appropriate time (according to fps).
-
- Returns
- -------
- `np.ndarray`
- image as a numpy array
- float
- timestamp at which frame was taken, obtained from :func:`time.time`
- """
-
- frame = None
- while frame is None:
- cur_time = time.time()
- if cur_time > self.next_frame:
- frame = self.get_image()
- timestamp = cur_time
- self.next_frame = max(
- self.next_frame + 1.0 / self.fps, cur_time + 0.5 / self.fps
- )
-
- return frame, timestamp
-
- def get_image(self):
- """ Gets image from frame capture device
- """
-
- raise NotImplementedError
-
- def close_capture_device(self):
- """ Closes frame capture device
- """
-
- raise NotImplementedError
diff --git a/dlclivegui/camera/opencv.py b/dlclivegui/camera/opencv.py
deleted file mode 100644
index 7ac96b2..0000000
--- a/dlclivegui/camera/opencv.py
+++ /dev/null
@@ -1,155 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import cv2
-from tkinter import filedialog
-from imutils import rotate_bound
-import time
-import platform
-
-from dlclivegui.camera import Camera, CameraError
-
-
-class OpenCVCam(Camera):
- @staticmethod
- def arg_restrictions():
- """ Returns a dictionary of arguments restrictions for DLCLiveGUI
- """
-
- cap = cv2.VideoCapture()
- devs = [-1]
- avail = True
- while avail:
- cur_index = devs[-1] + 1
- avail = cap.open(cur_index)
- if avail:
- devs.append(cur_index)
- cap.release()
-
- return {"device": devs, "display": [True, False]}
-
- def __init__(
- self,
- device=-1,
- file="",
- resolution=[640, 480],
- auto_exposure=0,
- exposure=0,
- gain=0,
- rotate=0,
- crop=None,
- fps=30,
- display=True,
- display_resize=1.0,
- ):
-
- if device != -1:
- if file:
- raise DLCLiveCameraError(
- "A device and file were provided to OpenCVCam. Must initialize an OpenCVCam with either a device id or a video file."
- )
-
- self.video = False
- id = int(device)
-
- else:
- if not file:
- file = filedialog.askopenfilename(
- title="Select video file for DLC-live-GUI"
- )
- if not file:
- raise DLCLiveCameraError(
- "Neither a device nor file were provided to OpenCVCam. Must initialize an OpenCVCam with either a device id or a video file."
- )
-
- self.video = True
- cap = cv2.VideoCapture(file)
- resolution = (
- cap.get(cv2.CAP_PROP_FRAME_WIDTH),
- cap.get(cv2.CAP_PROP_FRAME_HEIGHT),
- )
- fps = cap.get(cv2.CAP_PROP_FPS)
- del cap
- id = file
-
- super().__init__(
- id,
- resolution=resolution,
- exposure=exposure,
- rotate=rotate,
- crop=crop,
- fps=fps,
- use_tk_display=display,
- display_resize=display_resize,
- )
- self.auto_exposure = auto_exposure
- self.gain = gain
-
- def set_capture_device(self):
-
- if not self.video:
-
- self.cap = (
- cv2.VideoCapture(self.id, cv2.CAP_V4L)
- if platform.system() == "Linux"
- else cv2.VideoCapture(self.id)
- )
- ret, frame = self.cap.read()
-
- if self.im_size:
- self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.im_size[0])
- self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.im_size[1])
- if self.auto_exposure:
- self.cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, self.auto_exposure)
- if self.exposure:
- self.cap.set(cv2.CAP_PROP_EXPOSURE, self.exposure)
- if self.gain:
- self.cap.set(cv2.CAP_PROP_GAIN, self.gain)
- if self.fps:
- self.cap.set(cv2.CAP_PROP_FPS, self.fps)
-
- else:
-
- self.cap = cv2.VideoCapture(self.id)
-
- # self.im_size = (self.cap.get(cv2.CAP_PROP_FRAME_WIDTH), self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
- # self.fps = self.cap.get(cv2.CAP_PROP_FPS)
- self.last_cap_read = 0
-
- self.cv2_color = self.cap.get(cv2.CAP_PROP_MODE)
-
- return True
-
- def get_image_on_time(self):
-
- # if video, wait...
- if self.video:
- while time.time() - self.last_cap_read < (1.0 / self.fps):
- pass
-
- ret, frame = self.cap.read()
-
- if ret:
- if self.rotate:
- frame = rotate_bound(frame, self.rotate)
- if self.crop:
- frame = frame[self.crop[2] : self.crop[3], self.crop[0] : self.crop[1]]
-
- if frame.ndim == 3:
- if self.cv2_color == 1:
- frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
-
- self.last_cap_read = time.time()
-
- return frame, self.last_cap_read
- else:
- raise CameraError("OpenCV VideoCapture.read did not return an image!")
-
- def close_capture_device(self):
-
- self.cap.release()
diff --git a/dlclivegui/camera/pseye.py b/dlclivegui/camera/pseye.py
deleted file mode 100644
index 4c79065..0000000
--- a/dlclivegui/camera/pseye.py
+++ /dev/null
@@ -1,101 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import cv2
-from imutils import rotate_bound
-import numpy as np
-import pseyepy
-
-from dlclivegui.camera import Camera, CameraError
-
-
-class PSEyeCam(Camera):
- @staticmethod
- def arg_restrictions():
-
- return {
- "device": [i for i in range(pseyepy.cam_count())],
- "resolution": [[320, 240], [640, 480]],
- "fps": [30, 40, 50, 60, 75, 100, 125],
- "colour": [True, False],
- "auto_whitebalance": [True, False],
- }
-
- def __init__(
- self,
- device=0,
- resolution=[320, 240],
- exposure=100,
- gain=20,
- rotate=0,
- crop=None,
- fps=60,
- colour=False,
- auto_whitebalance=False,
- red_balance=125,
- blue_balance=125,
- green_balance=125,
- display=True,
- display_resize=1.0,
- ):
-
- super().__init__(
- device,
- resolution=resolution,
- exposure=exposure,
- gain=gain,
- rotate=rotate,
- crop=crop,
- fps=fps,
- use_tk_display=display,
- display_resize=display_resize,
- )
- self.colour = colour
- self.auto_whitebalance = auto_whitebalance
- self.red_balance = red_balance
- self.blue_balance = blue_balance
- self.green_balance = green_balance
-
- def set_capture_device(self):
-
- if self.im_size[0] == 320:
- res = pseyepy.Camera.RES_SMALL
- elif self.im_size[0] == 640:
- res = pseyepy.Camera.RES_LARGE
- else:
- raise CameraError(f"pseye resolution {self.im_size} not supported")
-
- self.cap = pseyepy.Camera(
- self.id,
- fps=self.fps,
- resolution=res,
- exposure=self.exposure,
- gain=self.gain,
- colour=self.colour,
- auto_whitebalance=self.auto_whitebalance,
- red_balance=self.red_balance,
- blue_balance=self.blue_balance,
- green_balance=self.green_balance,
- )
-
- return True
-
- def get_image_on_time(self):
-
- frame, _ = self.cap.read()
-
- if self.rotate != 0:
- frame = rotate_bound(frame, self.rotate)
- if self.crop:
- frame = frame[self.crop[2] : self.crop[3], self.crop[0] : self.crop[1]]
-
- return frame
-
- def close_capture_device(self):
-
- self.cap.end()
diff --git a/dlclivegui/camera/tiscamera_linux.py b/dlclivegui/camera/tiscamera_linux.py
deleted file mode 100644
index 5f0afd8..0000000
--- a/dlclivegui/camera/tiscamera_linux.py
+++ /dev/null
@@ -1,204 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import warnings
-import numpy as np
-import time
-
-import gi
-
-gi.require_version("Tcam", "0.1")
-gi.require_version("Gst", "1.0")
-from gi.repository import Tcam, Gst, GLib, GObject
-
-from dlclivegui.camera import Camera
-
-
-class TISCam(Camera):
-
- FRAME_RATE_OPTIONS = [15, 30, 60, 120, 240, 480]
- FRAME_RATE_FRACTIONS = ["15/1", "30/1", "60/1", "120/1", "5000000/20833", "480/1"]
- IM_FORMAT = (720, 540)
- ROTATE_OPTIONS = ["identity", "90r", "180", "90l", "horiz", "vert"]
-
- @staticmethod
- def arg_restrictions():
-
- if not Gst.is_initialized():
- Gst.init()
-
- source = Gst.ElementFactory.make("tcambin")
- return {
- "serial_number": source.get_device_serials(),
- "fps": TISCam.FRAME_RATE_OPTIONS,
- "rotate": TISCam.ROTATE_OPTIONS,
- "color": [True, False],
- "display": [True, False],
- }
-
- def __init__(
- self,
- serial_number="",
- resolution=[720, 540],
- exposure=0.005,
- rotate="identity",
- crop=None,
- fps=120,
- color=False,
- display=True,
- tk_resize=1.0,
- ):
-
- super().__init__(
- serial_number,
- resolution=resolution,
- exposure=exposure,
- rotate=rotate,
- crop=crop,
- fps=fps,
- use_tk_display=(not display),
- display_resize=tk_resize,
- )
- self.color = color
- self.display = display
- self.sample_locked = False
- self.new_sample = False
-
- def no_auto(self):
-
- self.cam.set_tcam_property("Exposure Auto", GObject.Value(bool, False))
-
- def set_exposure(self, val):
-
- val = 1 if val > 1 else val
- val = 0 if val < 0 else val
- self.cam.set_tcam_property("Exposure", val * 1e6)
-
- def set_crop(self, crop):
-
- if crop:
- self.gst_crop = self.gst_pipeline.get_by_name("crop")
- self.gst_crop.set_property("left", crop[0])
- self.gst_crop.set_property("right", TISCam.IM_FORMAT[0] - crop[1])
- self.gst_crop.set_property("top", crop[2])
- self.gst_crop.set_property("bottom", TISCam.IM_FORMAT[1] - crop[3])
- self.im_size = (crop[3] - crop[2], crop[1] - crop[0])
-
- def set_rotation(self, val):
-
- if val:
- self.gst_rotate = self.gst_pipeline.get_by_name("rotate")
- self.gst_rotate.set_property("video-direction", val)
-
- def set_sink(self):
-
- self.gst_sink = self.gst_pipeline.get_by_name("sink")
- self.gst_sink.set_property("max-buffers", 1)
- self.gst_sink.set_property("drop", 1)
- self.gst_sink.set_property("emit-signals", True)
- self.gst_sink.connect("new-sample", self.get_image)
-
- def setup_gst(self, serial_number, fps):
-
- if not Gst.is_initialized():
- Gst.init()
-
- fps_index = np.where(
- [int(fps) == int(opt) for opt in TISCam.FRAME_RATE_OPTIONS]
- )[0][0]
- fps_frac = TISCam.FRAME_RATE_FRACTIONS[fps_index]
- fmat = "BGRx" if self.color else "GRAY8"
-
- pipeline = (
- "tcambin name=cam "
- "! videocrop name=crop "
- "! videoflip name=rotate "
- "! video/x-raw,format={},framerate={} ".format(fmat, fps_frac)
- )
-
- if self.display:
- pipe_sink = (
- "! tee name=t "
- "t. ! queue ! videoconvert ! ximagesink "
- "t. ! queue ! appsink name=sink"
- )
- else:
- pipe_sink = "! appsink name=sink"
-
- pipeline += pipe_sink
-
- self.gst_pipeline = Gst.parse_launch(pipeline)
-
- self.cam = self.gst_pipeline.get_by_name("cam")
- self.cam.set_property("serial", serial_number)
-
- self.set_exposure(self.exposure)
- self.set_crop(self.crop)
- self.set_rotation(self.rotate)
- self.set_sink()
-
- def set_capture_device(self):
-
- self.setup_gst(self.id, self.fps)
- self.gst_pipeline.set_state(Gst.State.PLAYING)
-
- return True
-
- def get_image(self, sink):
-
- # wait for sample to unlock
- while self.sample_locked:
- pass
-
- try:
-
- self.sample = sink.get_property("last-sample")
- self._convert_image_to_numpy()
-
- except GLib.Error as e:
-
- warnings.warn("Error reading image :: {}".format(e))
-
- finally:
-
- return 0
-
- def _convert_image_to_numpy(self):
-
- self.sample_locked = True
-
- buffer = self.sample.get_buffer()
- struct = self.sample.get_caps().get_structure(0)
-
- height = struct.get_value("height")
- width = struct.get_value("width")
- fmat = struct.get_value("format")
- dtype = np.uint16 if fmat == "GRAY16_LE" else np.uint8
- ncolors = 1 if "GRAY" in fmat else 4
-
- self.frame = np.ndarray(
- shape=(height, width, ncolors),
- buffer=buffer.extract_dup(0, buffer.get_size()),
- dtype=dtype,
- )
-
- self.sample_locked = False
- self.new_sample = True
-
- def get_image_on_time(self):
-
- # wait for new sample
- while not self.new_sample:
- pass
- self.new_sample = False
-
- return self.frame, time.time()
-
- def close_capture_device(self):
-
- self.gst_pipeline.set_state(Gst.State.NULL)
diff --git a/dlclivegui/camera/tiscamera_windows.py b/dlclivegui/camera/tiscamera_windows.py
deleted file mode 100644
index bac6359..0000000
--- a/dlclivegui/camera/tiscamera_windows.py
+++ /dev/null
@@ -1,129 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-import time
-import cv2
-
-from dlclivegui.camera import Camera, CameraError
-from dlclivegui.camera.tisgrabber_windows import TIS_CAM
-
-
-class TISCam(Camera):
- @staticmethod
- def arg_restrictions():
-
- return {"serial_number": TIS_CAM().GetDevices(), "rotate": [0, 90, 180, 270]}
-
- def __init__(
- self,
- serial_number="",
- resolution=[720, 540],
- exposure=0.005,
- rotate=0,
- crop=None,
- fps=100,
- display=True,
- display_resize=1.0,
- ):
- """
- Params
- ------
- serial_number = string; serial number for imaging source camera
- crop = dict; contains ints named top, left, height, width for cropping
- default = None, uses default parameters specific to camera
- """
-
- if (rotate == 90) or (rotate == 270):
- resolution = [resolution[1], resolution[0]]
-
- super().__init__(
- serial_number,
- resolution=resolution,
- exposure=exposure,
- rotate=rotate,
- crop=crop,
- fps=fps,
- use_tk_display=display,
- display_resize=display_resize,
- )
- self.display = display
-
- def set_exposure(self):
-
- val = self.exposure
- val = 1 if val > 1 else val
- val = 0 if val < 0 else val
- self.cam.SetPropertyAbsoluteValue("Exposure", "Value", val)
-
- def get_exposure(self):
-
- exposure = [0]
- self.cam.GetPropertyAbsoluteValue("Exposure", "Value", exposure)
- return round(exposure[0], 3)
-
- # def set_crop(self):
-
- # crop = self.crop
-
- # if crop:
- # top = int(crop[0])
- # left = int(crop[2])
- # height = int(crop[1]-top)
- # width = int(crop[3]-left)
-
- # if not self.crop_filter:
- # self.crop_filter = self.cam.CreateFrameFilter(b'ROI')
- # self.cam.AddFrameFilter(self.crop_filter)
-
- # self.cam.FilterSetParameter(self.crop_filter, b'Top', top)
- # self.cam.FilterSetParameter(self.crop_filter, b'Left', left)
- # self.cam.FilterSetParameter(self.crop_filter, b'Height', height)
- # self.cam.FilterSetParameter(self.crop_filter, b'Width', width)
-
- def set_rotation(self):
-
- if not self.rotation_filter:
- self.rotation_filter = self.cam.CreateFrameFilter(b"Rotate Flip")
- self.cam.AddFrameFilter(self.rotation_filter)
- self.cam.FilterSetParameter(
- self.rotation_filter, b"Rotation Angle", self.rotate
- )
-
- def set_fps(self):
-
- self.cam.SetFrameRate(self.fps)
-
- def set_capture_device(self):
-
- self.cam = TIS_CAM()
- self.crop_filter = None
- self.rotation_filter = None
- self.set_rotation()
- # self.set_crop()
- self.set_fps()
- self.next_frame = time.time()
-
- self.cam.open(self.id)
- self.cam.SetContinuousMode(0)
- self.cam.StartLive(0)
-
- self.set_exposure()
-
- return True
-
- def get_image(self):
-
- self.cam.SnapImage()
- frame = self.cam.GetImageEx()
- frame = cv2.flip(frame, 0)
- if self.crop is not None:
- frame = frame[self.crop[0] : self.crop[1], self.crop[2] : self.crop[3]]
- return frame
-
- def close_capture_device(self):
-
- self.cam.StopLive()
diff --git a/dlclivegui/camera/tisgrabber_windows.py b/dlclivegui/camera/tisgrabber_windows.py
deleted file mode 100644
index 194e18e..0000000
--- a/dlclivegui/camera/tisgrabber_windows.py
+++ /dev/null
@@ -1,781 +0,0 @@
-"""
-Created on Mon Nov 21 09:44:40 2016
-
-@author: Daniel Vassmer, Stefan_Geissler
-From: https://github.com/TheImagingSource/IC-Imaging-Control-Samples/tree/master/Python
-
-modified 10/3/2019 by Gary Kane - https://github.com/gkane26
-"""
-
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-from enum import Enum
-
-import ctypes as C
-import os
-import sys
-import numpy as np
-
-
-class SinkFormats(Enum):
- Y800 = 0
- RGB24 = 1
- RGB32 = 2
- UYVY = 3
- Y16 = 4
-
-
-ImageFileTypes = {"BMP": 0, "JPEG": 1}
-
-
-class GrabberHandle(C.Structure):
- pass
-
-
-GrabberHandle._fields_ = [("unused", C.c_int)]
-
-##############################################################################
-
-### GK Additions from: https://github.com/morefigs/py-ic-imaging-control
-
-
-class FilterParameter(C.Structure):
- pass
-
-
-FilterParameter._fields_ = [("Name", C.c_char * 30), ("Type", C.c_int)]
-
-
-class FrameFilterHandle(C.Structure):
- pass
-
-
-FrameFilterHandle._fields_ = [
- ("pFilter", C.c_void_p),
- ("bHasDialog", C.c_int),
- ("ParameterCount", C.c_int),
- ("Parameters", C.POINTER(FilterParameter)),
-]
-
-##############################################################################
-
-
-class TIS_GrabberDLL(object):
- if sys.maxsize > 2 ** 32:
- __tisgrabber = C.windll.LoadLibrary("tisgrabber_x64.dll")
- else:
- __tisgrabber = C.windll.LoadLibrary("tisgrabber.dll")
-
- def __init__(self, **keyargs):
- """Initialize the Albatross from the keyword arguments."""
- self.__dict__.update(keyargs)
-
- GrabberHandlePtr = C.POINTER(GrabberHandle)
-
- ####################################
-
- # Initialize the ICImagingControl class library. This function must be called
- # only once before any other functions of this library are called.
- # @param szLicenseKey IC Imaging Control license key or NULL if only a trial version is available.
- # @retval IC_SUCCESS on success.
- # @retval IC_ERROR on wrong license key or other errors.
- # @sa IC_CloseLibrary
- InitLibrary = __tisgrabber.IC_InitLibrary(None)
-
- # Get the number of the currently available devices. This function creates an
- # internal array of all connected video capture devices. With each call to this
- # function, this array is rebuild. The name and the unique name can be retrieved
- # from the internal array using the functions IC_GetDevice() and IC_GetUniqueNamefromList.
- # They are usefull for retrieving device names for opening devices.
- #
- # @retval >= 0 Success, count of found devices.
- # @retval IC_NO_HANDLE Internal Error.
- #
- # @sa IC_GetDevice
- # @sa IC_GetUniqueNamefromList
- get_devicecount = __tisgrabber.IC_GetDeviceCount
- get_devicecount.restype = C.c_int
- get_devicecount.argtypes = None
-
- # Get unique device name of a device specified by iIndex. The unique device name
- # consist from the device name and its serial number. It allows to differ between
- # more then one device of the same type connected to the computer. The unique device name
- # is passed to the function IC_OpenDevByUniqueName
- #
- # @param iIndex The number of the device whose name is to be returned. It must be
- # in the range from 0 to IC_GetDeviceCount(),
- # @return Returns the string representation of the device on success, NULL
- # otherwise.
- #
- # @sa IC_GetDeviceCount
- # @sa IC_GetUniqueNamefromList
- # @sa IC_OpenDevByUniqueName
-
- get_unique_name_from_list = __tisgrabber.IC_GetUniqueNamefromList
- get_unique_name_from_list.restype = C.c_char_p
- get_unique_name_from_list.argtypes = (C.c_int,)
-
- # Creates a new grabber handle and returns it. A new created grabber should be
- # release with a call to IC_ReleaseGrabber if it is no longer needed.
- # @sa IC_ReleaseGrabber
- create_grabber = __tisgrabber.IC_CreateGrabber
- create_grabber.restype = GrabberHandlePtr
- create_grabber.argtypes = None
-
- # Open a video capture by using its UniqueName. Use IC_GetUniqueName() to
- # retrieve the unique name of a camera.
- #
- # @param hGrabber Handle to a grabber object
- # @param szDisplayName Memory that will take the display name.
- #
- # @sa IC_GetUniqueName
- # @sa IC_ReleaseGrabber
- open_device_by_unique_name = __tisgrabber.IC_OpenDevByUniqueName
- open_device_by_unique_name.restype = C.c_int
- open_device_by_unique_name.argtypes = (GrabberHandlePtr, C.c_char_p)
-
- set_videoformat = __tisgrabber.IC_SetVideoFormat
- set_videoformat.restype = C.c_int
- set_videoformat.argtypes = (GrabberHandlePtr, C.c_char_p)
-
- set_framerate = __tisgrabber.IC_SetFrameRate
- set_framerate.restype = C.c_int
- set_framerate.argtypes = (GrabberHandlePtr, C.c_float)
-
- # Returns the width of the video format.
- get_video_format_width = __tisgrabber.IC_GetVideoFormatWidth
- get_video_format_width.restype = C.c_int
- get_video_format_width.argtypes = (GrabberHandlePtr,)
-
- # returns the height of the video format.
- get_video_format_height = __tisgrabber.IC_GetVideoFormatHeight
- get_video_format_height.restype = C.c_int
- get_video_format_height.argtypes = (GrabberHandlePtr,)
-
- # Get the number of the available video formats for the current device.
- # A video capture device must have been opened before this call.
- #
- # @param hGrabber The handle to the grabber object.
- #
- # @retval >= 0 Success
- # @retval IC_NO_DEVICE No video capture device selected.
- # @retval IC_NO_HANDLE No handle to the grabber object.
- #
- # @sa IC_GetVideoFormat
- GetVideoFormatCount = __tisgrabber.IC_GetVideoFormatCount
- GetVideoFormatCount.restype = C.c_int
- GetVideoFormatCount.argtypes = (GrabberHandlePtr,)
-
- # Get a string representation of the video format specified by iIndex.
- # iIndex must be between 0 and IC_GetVideoFormatCount().
- # IC_GetVideoFormatCount() must have been called before this function,
- # otherwise it will always fail.
- #
- # @param hGrabber The handle to the grabber object.
- # @param iIndex Number of the video format to be used.
- #
- # @retval Nonnull The name of the specified video format.
- # @retval NULL An error occured.
- # @sa IC_GetVideoFormatCount
- GetVideoFormat = __tisgrabber.IC_GetVideoFormat
- GetVideoFormat.restype = C.c_char_p
- GetVideoFormat.argtypes = (GrabberHandlePtr, C.c_int)
-
- # Get the number of the available input channels for the current device.
- # A video capture device must have been opened before this call.
- #
- # @param hGrabber The handle to the grabber object.
- #
- # @retval >= 0 Success
- # @retval IC_NO_DEVICE No video capture device selected.
- # @retval IC_NO_HANDLE No handle to the grabber object.
- #
- # @sa IC_GetInputChannel
- GetInputChannelCount = __tisgrabber.IC_GetInputChannelCount
- GetInputChannelCount.restype = C.c_int
- GetInputChannelCount.argtypes = (GrabberHandlePtr,)
-
- # Get a string representation of the input channel specified by iIndex.
- # iIndex must be between 0 and IC_GetInputChannelCount().
- # IC_GetInputChannelCount() must have been called before this function,
- # otherwise it will always fail.
- # @param hGrabber The handle to the grabber object.
- # @param iIndex Number of the input channel to be used..
- #
- # @retval Nonnull The name of the specified input channel
- # @retval NULL An error occured.
- # @sa IC_GetInputChannelCount
- GetInputChannel = __tisgrabber.IC_GetInputChannel
- GetInputChannel.restype = C.c_char_p
- GetInputChannel.argtypes = (GrabberHandlePtr, C.c_int)
-
- # Get the number of the available video norms for the current device.
- # A video capture device must have been opened before this call.
- #
- # @param hGrabber The handle to the grabber object.
- #
- # @retval >= 0 Success
- # @retval IC_NO_DEVICE No video capture device selected.
- # @retval IC_NO_HANDLE No handle to the grabber object.
- #
- # @sa IC_GetVideoNorm
- GetVideoNormCount = __tisgrabber.IC_GetVideoNormCount
- GetVideoNormCount.restype = C.c_int
- GetVideoNormCount.argtypes = (GrabberHandlePtr,)
-
- # Get a string representation of the video norm specified by iIndex.
- # iIndex must be between 0 and IC_GetVideoNormCount().
- # IC_GetVideoNormCount() must have been called before this function,
- # otherwise it will always fail.
- #
- # @param hGrabber The handle to the grabber object.
- # @param iIndex Number of the video norm to be used.
- #
- # @retval Nonnull The name of the specified video norm.
- # @retval NULL An error occured.
- # @sa IC_GetVideoNormCount
- GetVideoNorm = __tisgrabber.IC_GetVideoNorm
- GetVideoNorm.restype = C.c_char_p
- GetVideoNorm.argtypes = (GrabberHandlePtr, C.c_int)
-
- SetFormat = __tisgrabber.IC_SetFormat
- SetFormat.restype = C.c_int
- SetFormat.argtypes = (GrabberHandlePtr, C.c_int)
- GetFormat = __tisgrabber.IC_GetFormat
- GetFormat.restype = C.c_int
- GetFormat.argtypes = (GrabberHandlePtr,)
-
- # Start the live video.
- # @param hGrabber The handle to the grabber object.
- # @param iShow The parameter indicates: @li 1 : Show the video @li 0 : Do not show the video, but deliver frames. (For callbacks etc.)
- # @retval IC_SUCCESS on success
- # @retval IC_ERROR if something went wrong.
- # @sa IC_StopLive
-
- StartLive = __tisgrabber.IC_StartLive
- StartLive.restype = C.c_int
- StartLive.argtypes = (GrabberHandlePtr, C.c_int)
-
- StopLive = __tisgrabber.IC_StopLive
- StopLive.restype = C.c_int
- StopLive.argtypes = (GrabberHandlePtr,)
-
- SetHWND = __tisgrabber.IC_SetHWnd
- SetHWND.restype = C.c_int
- SetHWND.argtypes = (GrabberHandlePtr, C.c_int)
-
- # Snaps an image. The video capture device must be set to live mode and a
- # sink type has to be set before this call. The format of the snapped images depend on
- # the selected sink type.
- #
- # @param hGrabber The handle to the grabber object.
- # @param iTimeOutMillisek The Timeout time is passed in milli seconds. A value of -1 indicates, that
- # no time out is set.
- #
- #
- # @retval IC_SUCCESS if an image has been snapped
- # @retval IC_ERROR if something went wrong.
- # @retval IC_NOT_IN_LIVEMODE if the live video has not been started.
- #
- # @sa IC_StartLive
- # @sa IC_SetFormat
-
- SnapImage = __tisgrabber.IC_SnapImage
- SnapImage.restype = C.c_int
- SnapImage.argtypes = (GrabberHandlePtr, C.c_int)
-
- # Retrieve the properties of the current video format and sink type
- # @param hGrabber The handle to the grabber object.
- # @param *lWidth This recieves the width of the image buffer.
- # @param *lHeight This recieves the height of the image buffer.
- # @param *iBitsPerPixel This recieves the count of bits per pixel.
- # @param *format This recieves the current color format.
- # @retval IC_SUCCESS on success
- # @retval IC_ERROR if something went wrong.
-
- GetImageDescription = __tisgrabber.IC_GetImageDescription
- GetImageDescription.restype = C.c_int
- GetImageDescription.argtypes = (
- GrabberHandlePtr,
- C.POINTER(C.c_long),
- C.POINTER(C.c_long),
- C.POINTER(C.c_int),
- C.POINTER(C.c_int),
- )
-
- GetImagePtr = __tisgrabber.IC_GetImagePtr
- GetImagePtr.restype = C.c_void_p
- GetImagePtr.argtypes = (GrabberHandlePtr,)
-
- # ############################################################################
- ShowDeviceSelectionDialog = __tisgrabber.IC_ShowDeviceSelectionDialog
- ShowDeviceSelectionDialog.restype = GrabberHandlePtr
- ShowDeviceSelectionDialog.argtypes = (GrabberHandlePtr,)
-
- # ############################################################################
-
- ShowPropertyDialog = __tisgrabber.IC_ShowPropertyDialog
- ShowPropertyDialog.restype = GrabberHandlePtr
- ShowPropertyDialog.argtypes = (GrabberHandlePtr,)
-
- # ############################################################################
- IsDevValid = __tisgrabber.IC_IsDevValid
- IsDevValid.restype = C.c_int
- IsDevValid.argtypes = (GrabberHandlePtr,)
-
- # ############################################################################
-
- LoadDeviceStateFromFile = __tisgrabber.IC_LoadDeviceStateFromFile
- LoadDeviceStateFromFile.restype = GrabberHandlePtr
- LoadDeviceStateFromFile.argtypes = (GrabberHandlePtr, C.c_char_p)
-
- # ############################################################################
- SaveDeviceStateToFile = __tisgrabber.IC_SaveDeviceStateToFile
- SaveDeviceStateToFile.restype = C.c_int
- SaveDeviceStateToFile.argtypes = (GrabberHandlePtr, C.c_char_p)
-
- GetCameraProperty = __tisgrabber.IC_GetCameraProperty
- GetCameraProperty.restype = C.c_int
- GetCameraProperty.argtypes = (GrabberHandlePtr, C.c_int, C.POINTER(C.c_long))
-
- SetCameraProperty = __tisgrabber.IC_SetCameraProperty
- SetCameraProperty.restype = C.c_int
- SetCameraProperty.argtypes = (GrabberHandlePtr, C.c_int, C.c_long)
-
- SetPropertyValue = __tisgrabber.IC_SetPropertyValue
- SetPropertyValue.restype = C.c_int
- SetPropertyValue.argtypes = (GrabberHandlePtr, C.c_char_p, C.c_char_p, C.c_int)
-
- GetPropertyValue = __tisgrabber.IC_GetPropertyValue
- GetPropertyValue.restype = C.c_int
- GetPropertyValue.argtypes = (
- GrabberHandlePtr,
- C.c_char_p,
- C.c_char_p,
- C.POINTER(C.c_long),
- )
-
- # ############################################################################
- SetPropertySwitch = __tisgrabber.IC_SetPropertySwitch
- SetPropertySwitch.restype = C.c_int
- SetPropertySwitch.argtypes = (GrabberHandlePtr, C.c_char_p, C.c_char_p, C.c_int)
-
- GetPropertySwitch = __tisgrabber.IC_GetPropertySwitch
- GetPropertySwitch.restype = C.c_int
- GetPropertySwitch.argtypes = (
- GrabberHandlePtr,
- C.c_char_p,
- C.c_char_p,
- C.POINTER(C.c_long),
- )
- # ############################################################################
-
- IsPropertyAvailable = __tisgrabber.IC_IsPropertyAvailable
- IsPropertyAvailable.restype = C.c_int
- IsPropertyAvailable.argtypes = (GrabberHandlePtr, C.c_char_p, C.c_char_p)
-
- PropertyOnePush = __tisgrabber.IC_PropertyOnePush
- PropertyOnePush.restype = C.c_int
- PropertyOnePush.argtypes = (GrabberHandlePtr, C.c_char_p, C.c_char_p)
-
- SetPropertyAbsoluteValue = __tisgrabber.IC_SetPropertyAbsoluteValue
- SetPropertyAbsoluteValue.restype = C.c_int
- SetPropertyAbsoluteValue.argtypes = (
- GrabberHandlePtr,
- C.c_char_p,
- C.c_char_p,
- C.c_float,
- )
-
- GetPropertyAbsoluteValue = __tisgrabber.IC_GetPropertyAbsoluteValue
- GetPropertyAbsoluteValue.restype = C.c_int
- GetPropertyAbsoluteValue.argtypes = (
- GrabberHandlePtr,
- C.c_char_p,
- C.c_char_p,
- C.POINTER(C.c_float),
- )
-
- # definition of the frameready callback
- FRAMEREADYCALLBACK = C.CFUNCTYPE(
- C.c_void_p, C.c_int, C.POINTER(C.c_ubyte), C.c_ulong, C.py_object
- )
-
- # set callback function
- SetFrameReadyCallback = __tisgrabber.IC_SetFrameReadyCallback
- SetFrameReadyCallback.restype = C.c_int
- SetFrameReadyCallback.argtypes = [GrabberHandlePtr, FRAMEREADYCALLBACK, C.py_object]
-
- SetContinuousMode = __tisgrabber.IC_SetContinuousMode
-
- SaveImage = __tisgrabber.IC_SaveImage
- SaveImage.restype = C.c_int
- SaveImage.argtypes = [C.c_void_p, C.c_char_p, C.c_int, C.c_int]
-
- OpenVideoCaptureDevice = __tisgrabber.IC_OpenVideoCaptureDevice
- OpenVideoCaptureDevice.restype = C.c_int
- OpenVideoCaptureDevice.argtypes = [C.c_void_p, C.c_char_p]
-
- # ############################################################################
-
- ### GK Additions - adding frame filters. Pieces copied from: https://github.com/morefigs/py-ic-imaging-control
-
- CreateFrameFilter = __tisgrabber.IC_CreateFrameFilter
- CreateFrameFilter.restype = C.c_int
- CreateFrameFilter.argtypes = (C.c_char_p, C.POINTER(FrameFilterHandle))
-
- AddFrameFilter = __tisgrabber.IC_AddFrameFilterToDevice
- AddFrameFilter.restype = C.c_int
- AddFrameFilter.argtypes = (GrabberHandlePtr, C.POINTER(FrameFilterHandle))
-
- FilterGetParameter = __tisgrabber.IC_FrameFilterGetParameter
- FilterGetParameter.restype = C.c_int
- FilterGetParameter.argtypes = (C.POINTER(FrameFilterHandle), C.c_char_p, C.c_void_p)
-
- FilterSetParameter = __tisgrabber.IC_FrameFilterSetParameterInt
- FilterSetParameter.restype = C.c_int
- FilterSetParameter.argtypes = (C.POINTER(FrameFilterHandle), C.c_char_p, C.c_int)
-
-
-# ############################################################################
-
-
-class TIS_CAM(object):
- @property
- def callback_registered(self):
- return self._callback_registered
-
- def __init__(self):
-
- self._handle = C.POINTER(GrabberHandle)
- self._handle = TIS_GrabberDLL.create_grabber()
- self._callback_registered = False
- self._frame = {"num": -1, "ready": False}
-
- def s(self, strin):
- if sys.version[0] == "2":
- return strin
- if type(strin) == "byte":
- return strin
- return strin.encode("utf-8")
-
- def SetFrameReadyCallback(self, CallbackFunction, data):
- """ Set a callback function, which is called, when a new frame arrives.
-
- CallbackFunction : The callback function
-
- data : a self defined class with user data.
- """
- return TIS_GrabberDLL.SetFrameReadyCallback(
- self._handle, CallbackFunction, data
- )
-
- def SetContinuousMode(self, Mode):
- """ Determines, whether new frames are automatically copied into memory.
-
- :param Mode: If 0, all frames are copied automatically into memory. This is recommened, if the camera runs in trigger mode.
- If 1, then snapImages must be called to get a frame into memory.
- :return: None
- """
- return TIS_GrabberDLL.SetContinuousMode(self._handle, Mode)
-
- def open(self, unique_device_name):
- """ Open a device
-
- unique_device_name : The name and serial number of the device to be opened. The device name and serial number are separated by a space.
- """
- test = TIS_GrabberDLL.open_device_by_unique_name(
- self._handle, self.s(unique_device_name)
- )
-
- return test
-
- def close(self):
- TIS_GrabberDLL.close_device(self._handle)
-
- def ShowDeviceSelectionDialog(self):
- self._handle = TIS_GrabberDLL.ShowDeviceSelectionDialog(self._handle)
-
- def ShowPropertyDialog(self):
- self._handle = TIS_GrabberDLL.ShowPropertyDialog(self._handle)
-
- def IsDevValid(self):
- return TIS_GrabberDLL.IsDevValid(self._handle)
-
- def SetHWND(self, Hwnd):
- return TIS_GrabberDLL.SetHWND(self._handle, Hwnd)
-
- def SaveDeviceStateToFile(self, FileName):
- return TIS_GrabberDLL.SaveDeviceStateToFile(self._handle, self.s(FileName))
-
- def LoadDeviceStateFromFile(self, FileName):
- self._handle = TIS_GrabberDLL.LoadDeviceStateFromFile(
- self._handle, self.s(FileName)
- )
-
- def SetVideoFormat(self, Format):
- return TIS_GrabberDLL.set_videoformat(self._handle, self.s(Format))
-
- def SetFrameRate(self, FPS):
- return TIS_GrabberDLL.set_framerate(self._handle, FPS)
-
- def get_video_format_width(self):
- return TIS_GrabberDLL.get_video_format_width(self._handle)
-
- def get_video_format_height(self):
- return TIS_GrabberDLL.get_video_format_height(self._handle)
-
- def GetDevices(self):
- self._Devices = []
- iDevices = TIS_GrabberDLL.get_devicecount()
- for i in range(iDevices):
- self._Devices.append(TIS_GrabberDLL.get_unique_name_from_list(i))
- return self._Devices
-
- def GetVideoFormats(self):
- self._Properties = []
- iVideoFormats = TIS_GrabberDLL.GetVideoFormatCount(self._handle)
- for i in range(iVideoFormats):
- self._Properties.append(TIS_GrabberDLL.GetVideoFormat(self._handle, i))
- return self._Properties
-
- def GetInputChannels(self):
- self.InputChannels = []
- InputChannelscount = TIS_GrabberDLL.GetInputChannelCount(self._handle)
- for i in range(InputChannelscount):
- self.InputChannels.append(TIS_GrabberDLL.GetInputChannel(self._handle, i))
- return self.InputChannels
-
- def GetVideoNormCount(self):
- self.GetVideoNorm = []
- GetVideoNorm_Count = TIS_GrabberDLL.GetVideoNormCount(self._handle)
- for i in range(GetVideoNorm_Count):
- self.GetVideoNorm.append(TIS_GrabberDLL.GetVideoNorm(self._handle, i))
- return self.GetVideoNorm
-
- def SetFormat(self, Format):
- """ SetFormat
- Sets the pixel format in memory
- @param Format Sinkformat enumeration
- """
- TIS_GrabberDLL.SetFormat(self._handle, Format.value)
-
- def GetFormat(self):
- val = TIS_GrabberDLL.GetFormat(self._handle)
- if val == 0:
- return SinkFormats.Y800
- if val == 2:
- return SinkFormats.RGB32
- if val == 1:
- return SinkFormats.RGB24
- if val == 3:
- return SinkFormats.UYVY
- if val == 4:
- return SinkFormats.Y16
- return SinkFormats.RGB24
-
- def StartLive(self, showlive=1):
- """
- Start the live video stream.
-
- showlive: 1 : a live video is shown, 0 : the live video is not shown.
- """
- Error = TIS_GrabberDLL.StartLive(self._handle, showlive)
- return Error
-
- def StopLive(self):
- """
- Stop the live video.
- """
- Error = TIS_GrabberDLL.StopLive(self._handle)
- return Error
-
- def SnapImage(self):
- Error = TIS_GrabberDLL.SnapImage(self._handle, 2000)
- return Error
-
- def GetImageDescription(self):
- lWidth = C.c_long()
- lHeight = C.c_long()
- iBitsPerPixel = C.c_int()
- COLORFORMAT = C.c_int()
-
- Error = TIS_GrabberDLL.GetImageDescription(
- self._handle, lWidth, lHeight, iBitsPerPixel, COLORFORMAT
- )
- return (lWidth.value, lHeight.value, iBitsPerPixel.value, COLORFORMAT.value)
-
- def GetImagePtr(self):
- ImagePtr = TIS_GrabberDLL.GetImagePtr(self._handle)
-
- return ImagePtr
-
- def GetImage(self):
- BildDaten = self.GetImageDescription()[:4]
- lWidth = BildDaten[0]
- lHeight = BildDaten[1]
- iBitsPerPixel = BildDaten[2] // 8
-
- buffer_size = lWidth * lHeight * iBitsPerPixel * C.sizeof(C.c_uint8)
- img_ptr = self.GetImagePtr()
-
- Bild = C.cast(img_ptr, C.POINTER(C.c_ubyte * buffer_size))
-
- img = np.ndarray(
- buffer=Bild.contents, dtype=np.uint8, shape=(lHeight, lWidth, iBitsPerPixel)
- )
- return img
-
- def GetImageEx(self):
- """ Return a numpy array with the image data tyes
- If the sink is Y16 or RGB64 (not supported yet), the dtype in the array is uint16, othereise it is uint8
- """
- BildDaten = self.GetImageDescription()[:4]
- lWidth = BildDaten[0]
- lHeight = BildDaten[1]
- iBytesPerPixel = BildDaten[2] // 8
-
- buffer_size = lWidth * lHeight * iBytesPerPixel * C.sizeof(C.c_uint8)
- img_ptr = self.GetImagePtr()
-
- Bild = C.cast(img_ptr, C.POINTER(C.c_ubyte * buffer_size))
-
- pixeltype = np.uint8
-
- if BildDaten[3] == 4: # SinkFormats.Y16:
- pixeltype = np.uint16
- iBytesPerPixel = 1
-
- img = np.ndarray(
- buffer=Bild.contents,
- dtype=pixeltype,
- shape=(lHeight, lWidth, iBytesPerPixel),
- )
- return img
-
- def GetCameraProperty(self, iProperty):
- lFocusPos = C.c_long()
- Error = TIS_GrabberDLL.GetCameraProperty(self._handle, iProperty, lFocusPos)
- return lFocusPos.value
-
- def SetCameraProperty(self, iProperty, iValue):
- Error = TIS_GrabberDLL.SetCameraProperty(self._handle, iProperty, iValue)
- return Error
-
- def SetPropertyValue(self, Property, Element, Value):
- error = TIS_GrabberDLL.SetPropertyValue(
- self._handle, self.s(Property), self.s(Element), Value
- )
- return error
-
- def GetPropertyValue(self, Property, Element):
- Value = C.c_long()
- error = TIS_GrabberDLL.GetPropertyValue(
- self._handle, self.s(Property), self.s(Element), Value
- )
- return Value.value
-
- def PropertyAvailable(self, Property):
- Null = None
- error = TIS_GrabberDLL.IsPropertyAvailable(self._handle, self.s(Property), Null)
- return error
-
- def SetPropertySwitch(self, Property, Element, Value):
- error = TIS_GrabberDLL.SetPropertySwitch(
- self._handle, self.s(Property), self.s(Element), Value
- )
- return error
-
- def GetPropertySwitch(self, Property, Element, Value):
- lValue = C.c_long()
- error = TIS_GrabberDLL.GetPropertySwitch(
- self._handle, self.s(Property), self.s(Element), lValue
- )
- Value[0] = lValue.value
- return error
-
- def PropertyOnePush(self, Property, Element):
- error = TIS_GrabberDLL.PropertyOnePush(
- self._handle, self.s(Property), self.s(Element)
- )
- return error
-
- def SetPropertyAbsoluteValue(self, Property, Element, Value):
- error = TIS_GrabberDLL.SetPropertyAbsoluteValue(
- self._handle, self.s(Property), self.s(Element), Value
- )
- return error
-
- def GetPropertyAbsoluteValue(self, Property, Element, Value):
- """ Get a property value of absolute values interface, e.g. seconds or dB.
- Example code:
- ExposureTime=[0]
- Camera.GetPropertyAbsoluteValue("Exposure","Value", ExposureTime)
- print("Exposure time in secods: ", ExposureTime[0])
-
- :param Property: Name of the property, e.g. Gain, Exposure
- :param Element: Name of the element, e.g. "Value"
- :param Value: Object, that receives the value of the property
- :returns: 0 on success
- """
- lValue = C.c_float()
- error = TIS_GrabberDLL.GetPropertyAbsoluteValue(
- self._handle, self.s(Property), self.s(Element), lValue
- )
- Value[0] = lValue.value
- return error
-
- def SaveImage(self, FileName, FileType, Quality=75):
- """ Saves the last snapped image. Can by of type BMP or JPEG.
- :param FileName : Name of the mage file
- :param FileType : Determines file type, can be "JPEG" or "BMP"
- :param Quality : If file typ is JPEG, the qualitly can be given from 1 to 100.
- :return: Error code
- """
- return TIS_GrabberDLL.SaveImage(
- self._handle, self.s(FileName), IC.ImageFileTypes[self.s(FileType)], Quality
- )
-
- def openVideoCaptureDevice(self, DeviceName):
- """ Open the device specified by DeviceName
- :param DeviceName: Name of the device , e.g. "DFK 72AUC02"
- :returns: 1 on success, 0 otherwise.
- """
- return TIS_GrabberDLL.OpenVideoCaptureDevice(self._handle, self.s(DeviceName))
-
- def CreateFrameFilter(self, name):
- frame_filter_handle = FrameFilterHandle()
-
- err = TIS_GrabberDLL.CreateFrameFilter(
- C.c_char_p(name), C.byref(frame_filter_handle)
- )
- if err != 1:
- raise Exception("ERROR CREATING FILTER")
- return frame_filter_handle
-
- def AddFrameFilter(self, frame_filter_handle):
- err = TIS_GrabberDLL.AddFrameFilter(self._handle, frame_filter_handle)
- return err
-
- def FilterGetParameter(self, frame_filter_handle, parameter_name):
- data = C.c_int()
-
- err = TIS_GrabberDLL.FilterGetParameter(
- frame_filter_handle, parameter_name, C.byref(data)
- )
- return data.value
-
- def FilterSetParameter(self, frame_filter_handle, parameter_name, data):
- if type(data) is int:
- err = TIS_GrabberDLL.FilterSetParameter(
- frame_filter_handle, C.c_char_p(parameter_name), C.c_int(data)
- )
- return err
- else:
- raise Exception("Unknown set parameter type")
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
new file mode 100644
index 0000000..3398566
--- /dev/null
+++ b/dlclivegui/camera_controller.py
@@ -0,0 +1,120 @@
+"""Camera management for the DLC Live GUI."""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Optional
+
+import numpy as np
+from PyQt6.QtCore import QMetaObject, QObject, QThread, Qt, pyqtSignal, pyqtSlot
+
+from .cameras import CameraFactory
+from .cameras.base import CameraBackend
+from .config import CameraSettings
+
+
+@dataclass
+class FrameData:
+ """Container for a captured frame."""
+
+ image: np.ndarray
+ timestamp: float
+
+
+class CameraWorker(QObject):
+ """Worker object running inside a :class:`QThread`."""
+
+ frame_captured = pyqtSignal(object)
+ error_occurred = pyqtSignal(str)
+ finished = pyqtSignal()
+
+ def __init__(self, settings: CameraSettings):
+ super().__init__()
+ self._settings = settings
+ self._running = False
+ self._backend: Optional[CameraBackend] = None
+
+ @pyqtSlot()
+ def run(self) -> None:
+ self._running = True
+ try:
+ self._backend = CameraFactory.create(self._settings)
+ self._backend.open()
+ except Exception as exc: # pragma: no cover - device specific
+ self.error_occurred.emit(str(exc))
+ self.finished.emit()
+ return
+
+ while self._running:
+ try:
+ frame, timestamp = self._backend.read()
+ except Exception as exc: # pragma: no cover - device specific
+ self.error_occurred.emit(str(exc))
+ break
+ self.frame_captured.emit(FrameData(frame, timestamp))
+
+ if self._backend is not None:
+ try:
+ self._backend.close()
+ except Exception as exc: # pragma: no cover - device specific
+ self.error_occurred.emit(str(exc))
+ self._backend = None
+ self.finished.emit()
+
+ @pyqtSlot()
+ def stop(self) -> None:
+ self._running = False
+ if self._backend is not None:
+ try:
+ self._backend.stop()
+ except Exception:
+ pass
+
+
+class CameraController(QObject):
+ """High level controller that manages a camera worker thread."""
+
+ frame_ready = pyqtSignal(object)
+ started = pyqtSignal(CameraSettings)
+ stopped = pyqtSignal()
+ error = pyqtSignal(str)
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._thread: Optional[QThread] = None
+ self._worker: Optional[CameraWorker] = None
+
+ def is_running(self) -> bool:
+ return self._thread is not None and self._thread.isRunning()
+
+ def start(self, settings: CameraSettings) -> None:
+ if self.is_running():
+ self.stop()
+ self._thread = QThread()
+ self._worker = CameraWorker(settings)
+ self._worker.moveToThread(self._thread)
+ self._thread.started.connect(self._worker.run)
+ self._worker.frame_captured.connect(self.frame_ready)
+ self._worker.error_occurred.connect(self.error)
+ self._worker.finished.connect(self._thread.quit)
+ self._worker.finished.connect(self._worker.deleteLater)
+ self._thread.finished.connect(self._cleanup)
+ self._thread.start()
+ self.started.emit(settings)
+
+ def stop(self) -> None:
+ if not self.is_running():
+ return
+ assert self._worker is not None
+ QMetaObject.invokeMethod(
+ self._worker, "stop", Qt.ConnectionType.QueuedConnection
+ )
+ assert self._thread is not None
+ self._thread.quit()
+ self._thread.wait()
+
+ @pyqtSlot()
+ def _cleanup(self) -> None:
+ self._thread = None
+ self._worker = None
+ self.stopped.emit()
diff --git a/dlclivegui/camera_process.py b/dlclivegui/camera_process.py
deleted file mode 100644
index 324c8e7..0000000
--- a/dlclivegui/camera_process.py
+++ /dev/null
@@ -1,338 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import time
-import multiprocess as mp
-import ctypes
-from dlclivegui.queue import ClearableQueue, ClearableMPQueue
-import threading
-import cv2
-import numpy as np
-import os
-
-
-class CameraProcessError(Exception):
- """
- Exception for incorrect use of Cameras
- """
-
- pass
-
-
-class CameraProcess(object):
- """ Camera Process Manager class. Controls image capture and writing images to a video file in a background process.
-
- Parameters
- ----------
- device : :class:`cameracontrol.Camera`
- a camera object
- ctx : :class:`multiprocess.Context`
- multiprocessing context
- """
-
- def __init__(self, device, ctx=mp.get_context("spawn")):
- """ Constructor method
- """
-
- self.device = device
- self.ctx = ctx
-
- res = self.device.im_size
- self.frame_shared = mp.Array(ctypes.c_uint8, res[1] * res[0] * 3)
- self.frame = np.frombuffer(self.frame_shared.get_obj(), dtype="uint8").reshape(
- res[1], res[0], 3
- )
- self.frame_time_shared = mp.Array(ctypes.c_double, 1)
- self.frame_time = np.frombuffer(self.frame_time_shared.get_obj(), dtype="d")
-
- self.q_to_process = ClearableMPQueue(ctx=self.ctx)
- self.q_from_process = ClearableMPQueue(ctx=self.ctx)
- self.write_frame_queue = ClearableMPQueue(ctx=self.ctx)
-
- self.capture_process = None
- self.writer_process = None
-
- def start_capture_process(self, timeout=60):
-
- cmds = self.q_to_process.read(clear=True, position="all")
- if cmds is not None:
- for c in cmds:
- if c[1] != "capture":
- self.q_to_process.write(c)
-
- self.capture_process = self.ctx.Process(
- target=self._run_capture,
- args=(self.frame_shared, self.frame_time_shared),
- daemon=True,
- )
- self.capture_process.start()
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "capture") and (cmd[1] == "start"):
- return cmd[2]
- else:
- self.q_to_process.write(cmd)
-
- return True
-
- def _run_capture(self, frame_shared, frame_time):
-
- res = self.device.im_size
- self.frame = np.frombuffer(frame_shared.get_obj(), dtype="uint8").reshape(
- res[1], res[0], 3
- )
- self.frame_time = np.frombuffer(frame_time.get_obj(), dtype="d")
-
- ret = self.device.set_capture_device()
- if not ret:
- raise CameraProcessError("Could not start capture device.")
- self.q_from_process.write(("capture", "start", ret))
-
- self._capture_loop()
-
- self.device.close_capture_device()
- self.q_from_process.write(("capture", "end", True))
-
- def _capture_loop(self):
- """ Acquires frames from frame capture device in a loop
- """
-
- run = True
- write = False
- last_frame_time = time.time()
-
- while run:
-
- start_capture = time.time()
-
- frame, frame_time = self.device.get_image_on_time()
-
- write_capture = time.time()
-
- np.copyto(self.frame, frame)
- self.frame_time[0] = frame_time
-
- if write:
- ret = self.write_frame_queue.write((frame, frame_time))
-
- end_capture = time.time()
-
- # print("read frame = %0.6f // write to queues = %0.6f" % (write_capture-start_capture, end_capture-write_capture))
- # print("capture rate = %d" % (int(1 / (time.time()-last_frame_time))))
- # print("\n")
-
- last_frame_time = time.time()
-
- ### read commands
- cmd = self.q_to_process.read()
- if cmd is not None:
- if cmd[0] == "capture":
- if cmd[1] == "write":
- write = cmd[2]
- self.q_from_process.write(cmd)
- elif cmd[1] == "end":
- run = False
- else:
- self.q_to_process.write(cmd)
-
- def stop_capture_process(self):
-
- ret = True
- if self.capture_process is not None:
- if self.capture_process.is_alive():
- self.q_to_process.write(("capture", "end"))
-
- while True:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if cmd[0] == "capture":
- if cmd[1] == "end":
- break
- else:
- self.q_from_process.write(cmd)
-
- self.capture_process.join(5)
- if self.capture_process.is_alive():
- self.capture_process.terminate()
-
- return True
-
- def start_writer_process(self, filename, timeout=60):
-
- cmds = self.q_to_process.read(clear=True, position="all")
- if cmds is not None:
- for c in cmds:
- if c[1] != "writer":
- self.q_to_process.write(c)
-
- self.writer_process = self.ctx.Process(
- target=self._run_writer, args=(filename,), daemon=True
- )
- self.writer_process.start()
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "writer") and (cmd[1] == "start"):
- return cmd[2]
- else:
- self.q_to_process.write(cmd)
-
- return True
-
- def _run_writer(self, filename):
-
- ret = self._create_writer(filename)
- self.q_from_process.write(("writer", "start", ret))
-
- save = self._write_loop()
-
- ret = self._save_video(not save)
- self.q_from_process.write(("writer", "end", ret))
-
- def _create_writer(self, filename):
-
- self.filename = filename
- self.video_file = f"{self.filename}_VIDEO.avi"
- self.timestamp_file = f"{self.filename}_TS.npy"
-
- self.video_writer = cv2.VideoWriter(
- self.video_file,
- cv2.VideoWriter_fourcc(*"DIVX"),
- self.device.fps,
- self.device.im_size,
- )
- self.write_frame_ts = []
-
- return True
-
- def _write_loop(self):
- """ read frames from write_frame_queue and write to file
- """
-
- run = True
- new_frame = None
-
- while run or (new_frame is not None):
-
- new_frame = self.write_frame_queue.read()
- if new_frame is not None:
- frame, ts = new_frame
- if frame.shape[2] == 1:
- frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
- self.video_writer.write(frame)
- self.write_frame_ts.append(ts)
-
- cmd = self.q_to_process.read()
- if cmd is not None:
- if cmd[0] == "writer":
- if cmd[1] == "end":
- run = False
- save = cmd[2]
- else:
- self.q_to_process.write(cmd)
-
- return save
-
- def _save_video(self, delete=False):
-
- ret = False
-
- self.video_writer.release()
-
- if (not delete) and (len(self.write_frame_ts) > 0):
- np.save(self.timestamp_file, self.write_frame_ts)
- ret = True
- else:
- os.remove(self.video_file)
- if os.path.isfile(self.timestamp_file):
- os.remove(self.timestamp_file)
-
- return ret
-
- def stop_writer_process(self, save=True):
-
- ret = False
- if self.writer_process is not None:
- if self.writer_process.is_alive():
- self.q_to_process.write(("writer", "end", save))
-
- while True:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if cmd[0] == "writer":
- if cmd[1] == "end":
- ret = cmd[2]
- break
- else:
- self.q_from_process.write(cmd)
-
- self.writer_process.join(5)
- if self.writer_process.is_alive():
- self.writer_process.terminate()
-
- return ret
-
- def start_record(self, timeout=5):
-
- ret = False
-
- if (self.capture_process is not None) and (self.writer_process is not None):
- if self.capture_process.is_alive() and self.writer_process.is_alive():
- self.q_to_process.write(("capture", "write", True))
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "capture") and (cmd[1] == "write"):
- ret = cmd[2]
- break
- else:
- self.q_from_process.write(cmd)
-
- return ret
-
- def stop_record(self, timeout=5):
-
- ret = False
-
- if (self.capture_process is not None) and (self.writer_process is not None):
- if (self.capture_process.is_alive()) and (self.writer_process.is_alive()):
- self.q_to_process.write(("capture", "write", False))
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "capture") and (cmd[1] == "write"):
- ret = not cmd[2]
- break
- else:
- self.q_from_process.write(cmd)
-
- return ret
-
- def get_display_frame(self):
-
- frame = self.frame.copy()
- if frame is not None:
- if self.device.display_resize != 1:
- frame = cv2.resize(
- frame,
- (
- int(frame.shape[1] * self.device.display_resize),
- int(frame.shape[0] * self.device.display_resize),
- ),
- )
-
- return frame
diff --git a/dlclivegui/cameras/__init__.py b/dlclivegui/cameras/__init__.py
new file mode 100644
index 0000000..cf7f488
--- /dev/null
+++ b/dlclivegui/cameras/__init__.py
@@ -0,0 +1,6 @@
+"""Camera backend implementations and factory helpers."""
+from __future__ import annotations
+
+from .factory import CameraFactory
+
+__all__ = ["CameraFactory"]
diff --git a/dlclivegui/cameras/base.py b/dlclivegui/cameras/base.py
new file mode 100644
index 0000000..6ae79dc
--- /dev/null
+++ b/dlclivegui/cameras/base.py
@@ -0,0 +1,46 @@
+"""Abstract camera backend definitions."""
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Tuple
+
+import numpy as np
+
+from ..config import CameraSettings
+
+
+class CameraBackend(ABC):
+ """Abstract base class for camera backends."""
+
+ def __init__(self, settings: CameraSettings):
+ self.settings = settings
+
+ @classmethod
+ def name(cls) -> str:
+ """Return the backend identifier."""
+
+ return cls.__name__.lower()
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Return whether the backend can be used on this system."""
+
+ return True
+
+ def stop(self) -> None:
+ """Request a graceful stop."""
+
+ # Most backends do not require additional handling, but subclasses may
+ # override when they need to interrupt blocking reads.
+
+ @abstractmethod
+ def open(self) -> None:
+ """Open the capture device."""
+
+ @abstractmethod
+ def read(self) -> Tuple[np.ndarray, float]:
+ """Read a frame and return the image with a timestamp."""
+
+ @abstractmethod
+ def close(self) -> None:
+ """Release the capture device."""
diff --git a/dlclivegui/cameras/basler_backend.py b/dlclivegui/cameras/basler_backend.py
new file mode 100644
index 0000000..f9e2a15
--- /dev/null
+++ b/dlclivegui/cameras/basler_backend.py
@@ -0,0 +1,132 @@
+"""Basler camera backend implemented with :mod:`pypylon`."""
+from __future__ import annotations
+
+import time
+from typing import Optional, Tuple
+
+import numpy as np
+
+from .base import CameraBackend
+
+try: # pragma: no cover - optional dependency
+ from pypylon import pylon
+except Exception: # pragma: no cover - optional dependency
+ pylon = None # type: ignore
+
+
+class BaslerCameraBackend(CameraBackend):
+ """Capture frames from Basler cameras using the Pylon SDK."""
+
+ def __init__(self, settings):
+ super().__init__(settings)
+ self._camera: Optional["pylon.InstantCamera"] = None
+ self._converter: Optional["pylon.ImageFormatConverter"] = None
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return pylon is not None
+
+ def open(self) -> None:
+ if pylon is None: # pragma: no cover - optional dependency
+ raise RuntimeError(
+ "pypylon is required for the Basler backend but is not installed"
+ )
+ devices = self._enumerate_devices()
+ if not devices:
+ raise RuntimeError("No Basler cameras detected")
+ device = self._select_device(devices)
+ self._camera = pylon.InstantCamera(pylon.TlFactory.GetInstance().CreateDevice(device))
+ self._camera.Open()
+
+ exposure = self._settings_value("exposure", self.settings.properties)
+ if exposure is not None:
+ self._camera.ExposureTime.SetValue(float(exposure))
+ gain = self._settings_value("gain", self.settings.properties)
+ if gain is not None:
+ self._camera.Gain.SetValue(float(gain))
+ width = int(self.settings.properties.get("width", self.settings.width))
+ height = int(self.settings.properties.get("height", self.settings.height))
+ self._camera.Width.SetValue(width)
+ self._camera.Height.SetValue(height)
+ fps = self._settings_value("fps", self.settings.properties, fallback=self.settings.fps)
+ if fps is not None:
+ try:
+ self._camera.AcquisitionFrameRateEnable.SetValue(True)
+ self._camera.AcquisitionFrameRate.SetValue(float(fps))
+ except Exception:
+ # Some cameras expose different frame-rate features; ignore errors.
+ pass
+
+ self._camera.StartGrabbing(pylon.GrabStrategy_LatestImageOnly)
+ self._converter = pylon.ImageFormatConverter()
+ self._converter.OutputPixelFormat = pylon.PixelType_BGR8packed
+ self._converter.OutputBitAlignment = pylon.OutputBitAlignment_MsbAligned
+
+ def read(self) -> Tuple[np.ndarray, float]:
+ if self._camera is None or self._converter is None:
+ raise RuntimeError("Basler camera not opened")
+ try:
+ grab_result = self._camera.RetrieveResult(100, pylon.TimeoutHandling_ThrowException)
+ except Exception as exc:
+ raise RuntimeError(f"Failed to retrieve image from Basler camera: {exc}")
+ if not grab_result.GrabSucceeded():
+ grab_result.Release()
+ raise RuntimeError("Basler camera did not return an image")
+ image = self._converter.Convert(grab_result)
+ frame = image.GetArray()
+ grab_result.Release()
+ rotate = self._settings_value("rotate", self.settings.properties)
+ if rotate:
+ frame = self._rotate(frame, float(rotate))
+ crop = self.settings.properties.get("crop")
+ if isinstance(crop, (list, tuple)) and len(crop) == 4:
+ left, right, top, bottom = map(int, crop)
+ frame = frame[top:bottom, left:right]
+ return frame, time.time()
+
+ def close(self) -> None:
+ if self._camera is not None:
+ if self._camera.IsGrabbing():
+ self._camera.StopGrabbing()
+ if self._camera.IsOpen():
+ self._camera.Close()
+ self._camera = None
+ self._converter = None
+
+ def stop(self) -> None:
+ if self._camera is not None and self._camera.IsGrabbing():
+ try:
+ self._camera.StopGrabbing()
+ except Exception:
+ pass
+
+ def _enumerate_devices(self):
+ factory = pylon.TlFactory.GetInstance()
+ return factory.EnumerateDevices()
+
+ def _select_device(self, devices):
+ serial = self.settings.properties.get("serial") or self.settings.properties.get("serial_number")
+ if serial:
+ for device in devices:
+ if getattr(device, "GetSerialNumber", None) and device.GetSerialNumber() == serial:
+ return device
+ index = int(self.settings.index)
+ if index < 0 or index >= len(devices):
+ raise RuntimeError(
+ f"Camera index {index} out of range for {len(devices)} Basler device(s)"
+ )
+ return devices[index]
+
+ def _rotate(self, frame: np.ndarray, angle: float) -> np.ndarray:
+ try:
+ from imutils import rotate_bound # pragma: no cover - optional
+ except Exception as exc: # pragma: no cover - optional dependency
+ raise RuntimeError(
+ "Rotation requested for Basler camera but imutils is not installed"
+ ) from exc
+ return rotate_bound(frame, angle)
+
+ @staticmethod
+ def _settings_value(key: str, source: dict, fallback: Optional[float] = None):
+ value = source.get(key, fallback)
+ return None if value is None else value
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
new file mode 100644
index 0000000..e9704ef
--- /dev/null
+++ b/dlclivegui/cameras/factory.py
@@ -0,0 +1,70 @@
+"""Backend discovery and construction utilities."""
+from __future__ import annotations
+
+import importlib
+from typing import Dict, Iterable, Tuple, Type
+
+from ..config import CameraSettings
+from .base import CameraBackend
+
+
+_BACKENDS: Dict[str, Tuple[str, str]] = {
+ "opencv": ("dlclivegui.cameras.opencv_backend", "OpenCVCameraBackend"),
+ "basler": ("dlclivegui.cameras.basler_backend", "BaslerCameraBackend"),
+ "gentl": ("dlclivegui.cameras.gentl_backend", "GenTLCameraBackend"),
+}
+
+
+class CameraFactory:
+ """Create camera backend instances based on configuration."""
+
+ @staticmethod
+ def backend_names() -> Iterable[str]:
+ """Return the identifiers of all known backends."""
+
+ return tuple(_BACKENDS.keys())
+
+ @staticmethod
+ def available_backends() -> Dict[str, bool]:
+ """Return a mapping of backend names to availability flags."""
+
+ availability: Dict[str, bool] = {}
+ for name in _BACKENDS:
+ try:
+ backend_cls = CameraFactory._resolve_backend(name)
+ except RuntimeError:
+ availability[name] = False
+ continue
+ availability[name] = backend_cls.is_available()
+ return availability
+
+ @staticmethod
+ def create(settings: CameraSettings) -> CameraBackend:
+ """Instantiate a backend for ``settings``."""
+
+ backend_name = (settings.backend or "opencv").lower()
+ try:
+ backend_cls = CameraFactory._resolve_backend(backend_name)
+ except RuntimeError as exc: # pragma: no cover - runtime configuration
+ raise RuntimeError(f"Unknown camera backend '{backend_name}': {exc}") from exc
+ if not backend_cls.is_available():
+ raise RuntimeError(
+ f"Camera backend '{backend_name}' is not available. "
+ "Ensure the required drivers and Python packages are installed."
+ )
+ return backend_cls(settings)
+
+ @staticmethod
+ def _resolve_backend(name: str) -> Type[CameraBackend]:
+ try:
+ module_name, class_name = _BACKENDS[name]
+ except KeyError as exc:
+ raise RuntimeError("backend not registered") from exc
+ try:
+ module = importlib.import_module(module_name)
+ except ImportError as exc:
+ raise RuntimeError(str(exc)) from exc
+ backend_cls = getattr(module, class_name)
+ if not issubclass(backend_cls, CameraBackend): # pragma: no cover - safety
+ raise RuntimeError(f"Backend '{name}' does not implement CameraBackend")
+ return backend_cls
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
new file mode 100644
index 0000000..0d81294
--- /dev/null
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -0,0 +1,130 @@
+"""Generic GenTL backend implemented with Aravis."""
+from __future__ import annotations
+
+import ctypes
+import time
+from typing import Optional, Tuple
+
+import numpy as np
+
+from .base import CameraBackend
+
+try: # pragma: no cover - optional dependency
+ import gi
+
+ gi.require_version("Aravis", "0.6")
+ from gi.repository import Aravis
+except Exception: # pragma: no cover - optional dependency
+ gi = None # type: ignore
+ Aravis = None # type: ignore
+
+
+class GenTLCameraBackend(CameraBackend):
+ """Capture frames from cameras that expose a GenTL interface."""
+
+ def __init__(self, settings):
+ super().__init__(settings)
+ self._camera = None
+ self._stream = None
+ self._payload: Optional[int] = None
+
+ @classmethod
+ def is_available(cls) -> bool:
+ return Aravis is not None
+
+ def open(self) -> None:
+ if Aravis is None: # pragma: no cover - optional dependency
+ raise RuntimeError(
+ "Aravis (python-gi bindings) are required for the GenTL backend"
+ )
+ Aravis.update_device_list()
+ num_devices = Aravis.get_n_devices()
+ if num_devices == 0:
+ raise RuntimeError("No GenTL cameras detected")
+ device_id = self._select_device_id(num_devices)
+ self._camera = Aravis.Camera.new(device_id)
+ self._camera.set_exposure_time_auto(0)
+ self._camera.set_gain_auto(0)
+ exposure = self.settings.properties.get("exposure")
+ if exposure is not None:
+ self._set_exposure(float(exposure))
+ crop = self.settings.properties.get("crop")
+ if isinstance(crop, (list, tuple)) and len(crop) == 4:
+ self._set_crop(crop)
+ if self.settings.fps:
+ try:
+ self._camera.set_frame_rate(float(self.settings.fps))
+ except Exception:
+ pass
+ self._stream = self._camera.create_stream()
+ self._payload = self._camera.get_payload()
+ self._stream.push_buffer(Aravis.Buffer.new_allocate(self._payload))
+ self._camera.start_acquisition()
+
+ def read(self) -> Tuple[np.ndarray, float]:
+ if self._stream is None:
+ raise RuntimeError("GenTL stream not initialised")
+ buffer = None
+ while buffer is None:
+ buffer = self._stream.try_pop_buffer()
+ if buffer is None:
+ time.sleep(0.01)
+ frame = self._buffer_to_numpy(buffer)
+ self._stream.push_buffer(buffer)
+ return frame, time.time()
+
+ def close(self) -> None:
+ if self._camera is not None:
+ try:
+ self._camera.stop_acquisition()
+ except Exception:
+ pass
+ self._camera = None
+ self._stream = None
+ self._payload = None
+
+ def stop(self) -> None:
+ if self._camera is not None:
+ try:
+ self._camera.stop_acquisition()
+ except Exception:
+ pass
+
+ def _select_device_id(self, num_devices: int) -> str:
+ index = int(self.settings.index)
+ if index < 0 or index >= num_devices:
+ raise RuntimeError(
+ f"Camera index {index} out of range for {num_devices} GenTL device(s)"
+ )
+ return Aravis.get_device_id(index)
+
+ def _set_exposure(self, exposure: float) -> None:
+ if self._camera is None:
+ return
+ exposure = max(0.0, min(exposure, 1.0))
+ self._camera.set_exposure_time(exposure * 1e6)
+
+ def _set_crop(self, crop) -> None:
+ if self._camera is None:
+ return
+ left, right, top, bottom = map(int, crop)
+ width = right - left
+ height = bottom - top
+ self._camera.set_region(left, top, width, height)
+
+ def _buffer_to_numpy(self, buffer) -> np.ndarray:
+ pixel_format = buffer.get_image_pixel_format()
+ bits_per_pixel = (pixel_format >> 16) & 0xFF
+ if bits_per_pixel == 8:
+ int_pointer = ctypes.POINTER(ctypes.c_uint8)
+ else:
+ int_pointer = ctypes.POINTER(ctypes.c_uint16)
+ addr = buffer.get_data()
+ ptr = ctypes.cast(addr, int_pointer)
+ frame = np.ctypeslib.as_array(ptr, (buffer.get_image_height(), buffer.get_image_width()))
+ frame = frame.copy()
+ if frame.ndim < 3:
+ import cv2
+
+ frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
+ return frame
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
new file mode 100644
index 0000000..a64e3f1
--- /dev/null
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -0,0 +1,61 @@
+"""OpenCV based camera backend."""
+from __future__ import annotations
+
+import time
+from typing import Tuple
+
+import cv2
+import numpy as np
+
+from .base import CameraBackend
+
+
+class OpenCVCameraBackend(CameraBackend):
+ """Fallback backend using :mod:`cv2.VideoCapture`."""
+
+ def __init__(self, settings):
+ super().__init__(settings)
+ self._capture: cv2.VideoCapture | None = None
+
+ def open(self) -> None:
+ backend_flag = self._resolve_backend(self.settings.properties.get("api"))
+ self._capture = cv2.VideoCapture(int(self.settings.index), backend_flag)
+ if not self._capture.isOpened():
+ raise RuntimeError(
+ f"Unable to open camera index {self.settings.index} with OpenCV"
+ )
+ self._configure_capture()
+
+ def read(self) -> Tuple[np.ndarray, float]:
+ if self._capture is None:
+ raise RuntimeError("Camera has not been opened")
+ success, frame = self._capture.read()
+ if not success:
+ raise RuntimeError("Failed to read frame from OpenCV camera")
+ return frame, time.time()
+
+ def close(self) -> None:
+ if self._capture is not None:
+ self._capture.release()
+ self._capture = None
+
+ def _configure_capture(self) -> None:
+ if self._capture is None:
+ return
+ self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, float(self.settings.width))
+ self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, float(self.settings.height))
+ self._capture.set(cv2.CAP_PROP_FPS, float(self.settings.fps))
+ for prop, value in self.settings.properties.items():
+ if prop == "api":
+ continue
+ try:
+ prop_id = int(prop)
+ except (TypeError, ValueError):
+ continue
+ self._capture.set(prop_id, float(value))
+
+ def _resolve_backend(self, backend: str | None) -> int:
+ if backend is None:
+ return cv2.CAP_ANY
+ key = backend.upper()
+ return getattr(cv2, f"CAP_{key}", cv2.CAP_ANY)
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
new file mode 100644
index 0000000..da00c5f
--- /dev/null
+++ b/dlclivegui/config.py
@@ -0,0 +1,112 @@
+"""Configuration helpers for the DLC Live GUI."""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+import json
+
+
+@dataclass
+class CameraSettings:
+ """Configuration for a single camera device."""
+
+ name: str = "Camera 0"
+ index: int = 0
+ width: int = 640
+ height: int = 480
+ fps: float = 30.0
+ backend: str = "opencv"
+ properties: Dict[str, Any] = field(default_factory=dict)
+
+ def apply_defaults(self) -> "CameraSettings":
+ """Ensure width, height and fps are positive numbers."""
+
+ self.width = int(self.width) if self.width else 640
+ self.height = int(self.height) if self.height else 480
+ self.fps = float(self.fps) if self.fps else 30.0
+ return self
+
+
+@dataclass
+class DLCProcessorSettings:
+ """Configuration for DLCLive processing."""
+
+ model_path: str = ""
+ shuffle: Optional[int] = None
+ trainingsetindex: Optional[int] = None
+ processor: str = "cpu"
+ processor_args: Dict[str, Any] = field(default_factory=dict)
+ additional_options: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RecordingSettings:
+ """Configuration for video recording."""
+
+ enabled: bool = False
+ directory: str = str(Path.home() / "Videos" / "deeplabcut-live")
+ filename: str = "session.mp4"
+ container: str = "mp4"
+ options: Dict[str, Any] = field(default_factory=dict)
+
+ def output_path(self) -> Path:
+ """Return the absolute output path for recordings."""
+
+ directory = Path(self.directory).expanduser().resolve()
+ directory.mkdir(parents=True, exist_ok=True)
+ name = Path(self.filename)
+ if name.suffix:
+ filename = name
+ else:
+ filename = name.with_suffix(f".{self.container}")
+ return directory / filename
+
+
+@dataclass
+class ApplicationSettings:
+ """Top level application configuration."""
+
+ camera: CameraSettings = field(default_factory=CameraSettings)
+ dlc: DLCProcessorSettings = field(default_factory=DLCProcessorSettings)
+ recording: RecordingSettings = field(default_factory=RecordingSettings)
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
+ """Create an :class:`ApplicationSettings` from a dictionary."""
+
+ camera = CameraSettings(**data.get("camera", {})).apply_defaults()
+ dlc = DLCProcessorSettings(**data.get("dlc", {}))
+ recording = RecordingSettings(**data.get("recording", {}))
+ return cls(camera=camera, dlc=dlc, recording=recording)
+
+ def to_dict(self) -> Dict[str, Any]:
+ """Serialise the configuration to a dictionary."""
+
+ return {
+ "camera": asdict(self.camera),
+ "dlc": asdict(self.dlc),
+ "recording": asdict(self.recording),
+ }
+
+ @classmethod
+ def load(cls, path: Path | str) -> "ApplicationSettings":
+ """Load configuration from ``path``."""
+
+ file_path = Path(path).expanduser()
+ if not file_path.exists():
+ raise FileNotFoundError(f"Configuration file not found: {file_path}")
+ with file_path.open("r", encoding="utf-8") as handle:
+ data = json.load(handle)
+ return cls.from_dict(data)
+
+ def save(self, path: Path | str) -> None:
+ """Persist configuration to ``path``."""
+
+ file_path = Path(path).expanduser()
+ file_path.parent.mkdir(parents=True, exist_ok=True)
+ with file_path.open("w", encoding="utf-8") as handle:
+ json.dump(self.to_dict(), handle, indent=2)
+
+
+DEFAULT_CONFIG = ApplicationSettings()
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
new file mode 100644
index 0000000..5c199b8
--- /dev/null
+++ b/dlclivegui/dlc_processor.py
@@ -0,0 +1,123 @@
+"""DLCLive integration helpers."""
+from __future__ import annotations
+
+import logging
+import threading
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from typing import Any, Optional
+
+import numpy as np
+from PyQt6.QtCore import QObject, pyqtSignal
+
+from .config import DLCProcessorSettings
+
+LOGGER = logging.getLogger(__name__)
+
+try: # pragma: no cover - optional dependency
+ from dlclive import DLCLive # type: ignore
+except Exception: # pragma: no cover - handled gracefully
+ DLCLive = None # type: ignore[assignment]
+
+
+@dataclass
+class PoseResult:
+ pose: Optional[np.ndarray]
+ timestamp: float
+
+
+class DLCLiveProcessor(QObject):
+ """Background pose estimation using DLCLive."""
+
+ pose_ready = pyqtSignal(object)
+ error = pyqtSignal(str)
+ initialized = pyqtSignal(bool)
+
+ def __init__(self) -> None:
+ super().__init__()
+ self._settings = DLCProcessorSettings()
+ self._executor = ThreadPoolExecutor(max_workers=1)
+ self._dlc: Optional[DLCLive] = None
+ self._init_future: Optional[Future[Any]] = None
+ self._pending: Optional[Future[Any]] = None
+ self._lock = threading.Lock()
+
+ def configure(self, settings: DLCProcessorSettings) -> None:
+ self._settings = settings
+
+ def shutdown(self) -> None:
+ with self._lock:
+ if self._pending is not None:
+ self._pending.cancel()
+ self._pending = None
+ if self._init_future is not None:
+ self._init_future.cancel()
+ self._init_future = None
+ self._executor.shutdown(wait=False, cancel_futures=True)
+ self._dlc = None
+
+ def enqueue_frame(self, frame: np.ndarray, timestamp: float) -> None:
+ with self._lock:
+ if self._dlc is None and self._init_future is None:
+ self._init_future = self._executor.submit(
+ self._initialise_model, frame.copy(), timestamp
+ )
+ self._init_future.add_done_callback(self._on_initialised)
+ return
+ if self._dlc is None:
+ return
+ if self._pending is not None and not self._pending.done():
+ return
+ self._pending = self._executor.submit(
+ self._run_inference, frame.copy(), timestamp
+ )
+ self._pending.add_done_callback(self._on_pose_ready)
+
+ def _initialise_model(self, frame: np.ndarray, timestamp: float) -> bool:
+ if DLCLive is None:
+ raise RuntimeError(
+ "The 'dlclive' package is required for pose estimation. Install it to enable DLCLive support."
+ )
+ if not self._settings.model_path:
+ raise RuntimeError("No DLCLive model path configured.")
+ options = {
+ "model_path": self._settings.model_path,
+ "processor": self._settings.processor,
+ }
+ options.update(self._settings.additional_options)
+ if self._settings.shuffle is not None:
+ options["shuffle"] = self._settings.shuffle
+ if self._settings.trainingsetindex is not None:
+ options["trainingsetindex"] = self._settings.trainingsetindex
+ if self._settings.processor_args:
+ options["processor_config"] = {
+ "object": self._settings.processor,
+ **self._settings.processor_args,
+ }
+ model = DLCLive(**options)
+ model.init_inference(frame, frame_time=timestamp, record=False)
+ self._dlc = model
+ return True
+
+ def _on_initialised(self, future: Future[Any]) -> None:
+ try:
+ result = future.result()
+ self.initialized.emit(bool(result))
+ except Exception as exc: # pragma: no cover - runtime behaviour
+ LOGGER.exception("Failed to initialise DLCLive", exc_info=exc)
+ self.error.emit(str(exc))
+
+ def _run_inference(self, frame: np.ndarray, timestamp: float) -> PoseResult:
+ if self._dlc is None:
+ raise RuntimeError("DLCLive model not initialised")
+ pose = self._dlc.get_pose(frame, frame_time=timestamp, record=False)
+ return PoseResult(pose=pose, timestamp=timestamp)
+
+ def _on_pose_ready(self, future: Future[Any]) -> None:
+ try:
+ result = future.result()
+ except Exception as exc: # pragma: no cover - runtime behaviour
+ LOGGER.exception("Pose inference failed", exc_info=exc)
+ self.error.emit(str(exc))
+ return
+ self.pose_ready.emit(result)
diff --git a/dlclivegui/dlclivegui.py b/dlclivegui/dlclivegui.py
deleted file mode 100644
index 2a29d7d..0000000
--- a/dlclivegui/dlclivegui.py
+++ /dev/null
@@ -1,1498 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-from tkinter import (
- Tk,
- Toplevel,
- Label,
- Entry,
- Button,
- Radiobutton,
- Checkbutton,
- StringVar,
- IntVar,
- BooleanVar,
- filedialog,
- messagebox,
- simpledialog,
-)
-from tkinter.ttk import Combobox
-import os
-import sys
-import glob
-import json
-import datetime
-import inspect
-import importlib
-
-from PIL import Image, ImageTk, ImageDraw
-import colorcet as cc
-
-from dlclivegui import CameraPoseProcess
-from dlclivegui import processor
-from dlclivegui import camera
-from dlclivegui.tkutil import SettingsWindow
-
-
-class DLCLiveGUI(object):
- """ GUI to run DLC Live experiment
- """
-
- def __init__(self):
- """ Constructor method
- """
-
- ### check if documents path exists
-
- if not os.path.isdir(self.get_docs_path()):
- os.mkdir(self.get_docs_path())
- if not os.path.isdir(os.path.dirname(self.get_config_path(""))):
- os.mkdir(os.path.dirname(self.get_config_path("")))
-
- ### get configuration ###
-
- self.cfg_list = [
- os.path.splitext(os.path.basename(f))[0]
- for f in glob.glob(os.path.dirname(self.get_config_path("")) + "/*.json")
- ]
-
- ### initialize variables
-
- self.cam_pose_proc = None
- self.dlc_proc_params = None
-
- self.display_window = None
- self.display_cmap = None
- self.display_colors = None
- self.display_radius = None
- self.display_lik_thresh = None
-
- ### create GUI window ###
-
- self.createGUI()
-
- def get_docs_path(self):
- """ Get path to documents folder
-
- Returns
- -------
- str
- path to documents folder
- """
-
- return os.path.normpath(os.path.expanduser("~/Documents/DeepLabCut-live-GUI"))
-
- def get_config_path(self, cfg_name):
- """ Get path to configuration foler
-
- Parameters
- ----------
- cfg_name : str
- name of config file
-
- Returns
- -------
- str
- path to configuration file
- """
-
- return os.path.normpath(self.get_docs_path() + "/config/" + cfg_name + ".json")
-
- def get_config(self, cfg_name):
- """ Read configuration
-
- Parameters
- ----------
- cfg_name : str
- name of configuration
- """
-
- ### read configuration file ###
-
- self.cfg_file = self.get_config_path(cfg_name)
- if os.path.isfile(self.cfg_file):
- cfg = json.load(open(self.cfg_file))
- else:
- cfg = {}
-
- ### check config ###
-
- cfg["cameras"] = {} if "cameras" not in cfg else cfg["cameras"]
- cfg["processor_dir"] = (
- [] if "processor_dir" not in cfg else cfg["processor_dir"]
- )
- cfg["processor_args"] = (
- {} if "processor_args" not in cfg else cfg["processor_args"]
- )
- cfg["dlc_options"] = {} if "dlc_options" not in cfg else cfg["dlc_options"]
- cfg["dlc_display_options"] = (
- {} if "dlc_display_options" not in cfg else cfg["dlc_display_options"]
- )
- cfg["subjects"] = [] if "subjects" not in cfg else cfg["subjects"]
- cfg["directories"] = [] if "directories" not in cfg else cfg["directories"]
-
- self.cfg = cfg
-
- def change_config(self, event=None):
- """ Change configuration, update GUI menus
-
- Parameters
- ----------
- event : tkinter event, optional
- event , by default None
- """
-
- if self.cfg_name.get() == "Create New Config":
- new_name = simpledialog.askstring(
- "", "Please enter a name (no special characters).", parent=self.window
- )
- self.cfg_name.set(new_name)
- self.get_config(self.cfg_name.get())
-
- self.camera_entry["values"] = tuple(self.cfg["cameras"].keys()) + (
- "Add Camera",
- )
- self.camera_name.set("")
- self.dlc_proc_dir_entry["values"] = tuple(self.cfg["processor_dir"])
- self.dlc_proc_dir.set("")
- self.dlc_proc_name_entry["values"] = tuple()
- self.dlc_proc_name.set("")
- self.dlc_options_entry["values"] = tuple(self.cfg["dlc_options"].keys()) + (
- "Add DLC",
- )
- self.dlc_option.set("")
- self.subject_entry["values"] = tuple(self.cfg["subjects"])
- self.subject.set("")
- self.directory_entry["values"] = tuple(self.cfg["directories"])
- self.directory.set("")
-
- def remove_config(self):
- """ Remove configuration
- """
-
- cfg_name = self.cfg_name.get()
- delete_setup = messagebox.askyesnocancel(
- "Delete Config Permanently?",
- "Would you like to delete the configuration {} permanently (yes),\nremove the setup from the list for this session (no),\nor neither (cancel).".format(
- cfg_name
- ),
- parent=self.window,
- )
- if delete_setup is not None:
- if delete_setup:
- os.remove(self.get_config_path(cfg_name))
- self.cfg_list.remove(cfg_name)
- self.cfg_entry["values"] = tuple(self.cfg_list) + ("Create New Setup",)
- self.cfg_name.set("")
-
- def get_camera_names(self):
- """ Get camera names from configuration as a tuple
- """
-
- return tuple(self.cfg["cameras"].keys())
-
- def init_cam(self):
- """ Initialize camera
- """
-
- if self.cam_pose_proc is not None:
- messagebox.showerror(
- "Camera Exists",
- "Camera already exists! Please close current camera before initializing a new one.",
- )
- return
-
- this_cam = self.get_current_camera()
-
- if not this_cam:
-
- messagebox.showerror(
- "No Camera",
- "No camera selected. Please select a camera before initializing.",
- parent=self.window,
- )
-
- else:
-
- if this_cam["type"] == "Add Camera":
-
- self.add_camera_window()
- return
-
- else:
-
- self.cam_setup_window = Toplevel(self.window)
- self.cam_setup_window.title("Setting up camera...")
- Label(
- self.cam_setup_window, text="Setting up camera, please wait..."
- ).pack()
- self.cam_setup_window.update()
-
- cam_obj = getattr(camera, this_cam["type"])
- cam = cam_obj(**this_cam["params"])
- self.cam_pose_proc = CameraPoseProcess(cam)
- ret = self.cam_pose_proc.start_capture_process()
-
- if cam.use_tk_display:
- self.set_display_window()
-
- self.cam_setup_window.destroy()
-
- def get_current_camera(self):
- """ Get dictionary of the current camera
- """
-
- if self.camera_name.get():
- if self.camera_name.get() == "Add Camera":
- return {"type": "Add Camera"}
- else:
- return self.cfg["cameras"][self.camera_name.get()]
-
- def set_camera_param(self, key, value):
- """ Set a camera parameter
- """
-
- self.cfg["cameras"][self.camera_name.get()]["params"][key] = value
-
- def add_camera_window(self):
- """ Create gui to add a camera
- """
-
- add_cam = Tk()
- cur_row = 0
-
- Label(add_cam, text="Type: ").grid(sticky="w", row=cur_row, column=0)
- self.cam_type = StringVar(add_cam)
-
- cam_types = [c[0] for c in inspect.getmembers(camera, inspect.isclass)]
- cam_types = [c for c in cam_types if (c != "Camera") & ("Error" not in c)]
-
- type_entry = Combobox(add_cam, textvariable=self.cam_type, state="readonly")
- type_entry["values"] = tuple(cam_types)
- type_entry.current(0)
- type_entry.grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
-
- Label(add_cam, text="Name: ").grid(sticky="w", row=cur_row, column=0)
- self.new_cam_name = StringVar(add_cam)
- Entry(add_cam, textvariable=self.new_cam_name).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 1
-
- Button(
- add_cam, text="Add Camera", command=lambda: self.add_cam_to_list(add_cam)
- ).grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
-
- Button(add_cam, text="Cancel", command=add_cam.destroy).grid(
- sticky="nsew", row=cur_row, column=1
- )
-
- add_cam.mainloop()
-
- def add_cam_to_list(self, gui):
- """ Add new camera to the camera list
- """
-
- self.cfg["cameras"][self.new_cam_name.get()] = {
- "type": self.cam_type.get(),
- "params": {},
- }
- self.camera_name.set(self.new_cam_name.get())
- self.camera_entry["values"] = self.get_camera_names() + ("Add Camera",)
- self.save_config()
- # messagebox.showinfo("Camera Added", "Camera has been added to the dropdown menu. Please edit camera settings before initializing the new camera.", parent=gui)
- gui.destroy()
-
- def edit_cam_settings(self):
- """ GUI window to edit camera settings
- """
-
- arg_names, arg_vals, arg_dtypes, arg_restrict = self.get_cam_args()
-
- settings_window = Toplevel(self.window)
- settings_window.title("Camera Settings")
- cur_row = 0
- combobox_width = 15
-
- entry_vars = []
- for n, v in zip(arg_names, arg_vals):
-
- Label(settings_window, text=n + ": ").grid(row=cur_row, column=0)
-
- if type(v) is list:
- v = [str(x) if x is not None else "" for x in v]
- v = ", ".join(v)
- else:
- v = v if v is not None else ""
- entry_vars.append(StringVar(settings_window, value=str(v)))
-
- if n in arg_restrict.keys():
- restrict_vals = arg_restrict[n]
- if type(restrict_vals[0]) is list:
- restrict_vals = [
- ", ".join([str(i) for i in rv]) for rv in restrict_vals
- ]
- Combobox(
- settings_window,
- textvariable=entry_vars[-1],
- values=restrict_vals,
- state="readonly",
- width=combobox_width,
- ).grid(sticky="nsew", row=cur_row, column=1)
- else:
- Entry(settings_window, textvariable=entry_vars[-1]).grid(
- sticky="nsew", row=cur_row, column=1
- )
-
- cur_row += 1
-
- cur_row += 1
- Button(
- settings_window,
- text="Update",
- command=lambda: self.update_camera_settings(
- arg_names, entry_vars, arg_dtypes, settings_window
- ),
- ).grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
- Button(settings_window, text="Cancel", command=settings_window.destroy).grid(
- sticky="nsew", row=cur_row, column=1
- )
-
- _, row_count = settings_window.grid_size()
- for r in range(row_count):
- settings_window.grid_rowconfigure(r, minsize=20)
-
- settings_window.mainloop()
-
- def get_cam_args(self):
- """ Get arguments for the new camera
- """
-
- this_cam = self.get_current_camera()
- cam_obj = getattr(camera, this_cam["type"])
- arg_restrict = cam_obj.arg_restrictions()
-
- cam_args = inspect.getfullargspec(cam_obj)
- n_args = len(cam_args[0][1:])
- n_vals = len(cam_args[3])
- arg_names = []
- arg_vals = []
- arg_dtype = []
- for i in range(n_args):
- arg_names.append(cam_args[0][i + 1])
-
- if arg_names[i] in this_cam["params"].keys():
- val = this_cam["params"][arg_names[i]]
- else:
- val = None if i < n_args - n_vals else cam_args[3][n_vals - n_args + i]
- arg_vals.append(val)
-
- dt_val = val if i < n_args - n_vals else cam_args[3][n_vals - n_args + i]
- dt = type(dt_val) if type(dt_val) is not list else type(dt_val[0])
- arg_dtype.append(dt)
-
- return arg_names, arg_vals, arg_dtype, arg_restrict
-
- def update_camera_settings(self, names, entries, dtypes, gui):
- """ Update camera settings from values input in settings GUI
- """
-
- gui.destroy()
-
- for name, entry, dt in zip(names, entries, dtypes):
- val = entry.get()
- val = val.split(",")
- val = [v.strip() for v in val]
- try:
- if dt is bool:
- val = [True if v == "True" else False for v in val]
- else:
- val = [dt(v) if v else None for v in val]
- except TypeError:
- pass
- val = val if len(val) > 1 else val[0]
- self.set_camera_param(name, val)
-
- self.save_config()
-
- def set_display_window(self):
- """ Create a video display window
- """
-
- self.display_window = Toplevel(self.window)
- self.display_frame_label = Label(self.display_window)
- self.display_frame_label.pack()
- self.display_frame()
-
- def set_display_colors(self, bodyparts):
- """ Set colors for keypoints
-
- Parameters
- ----------
- bodyparts : int
- the number of keypoints
- """
-
- all_colors = getattr(cc, self.display_cmap)
- self.display_colors = all_colors[:: int(len(all_colors) / bodyparts)]
-
- def display_frame(self):
- """ Display a frame in display window
- """
-
- if self.cam_pose_proc and self.display_window:
-
- frame = self.cam_pose_proc.get_display_frame()
-
- if frame is not None:
-
- img = Image.fromarray(frame)
- if frame.ndim == 3:
- b, g, r = img.split()
- img = Image.merge("RGB", (r, g, b))
-
- pose = (
- self.cam_pose_proc.get_display_pose()
- if self.display_keypoints.get()
- else None
- )
-
- if pose is not None:
-
- im_size = (frame.shape[1], frame.shape[0])
-
- if not self.display_colors:
- self.set_display_colors(pose.shape[0])
-
- img_draw = ImageDraw.Draw(img)
-
- for i in range(pose.shape[0]):
- if pose[i, 2] > self.display_lik_thresh:
- try:
- x0 = (
- pose[i, 0] - self.display_radius
- if pose[i, 0] - self.display_radius > 0
- else 0
- )
- x1 = (
- pose[i, 0] + self.display_radius
- if pose[i, 0] + self.display_radius < im_size[1]
- else im_size[1]
- )
- y0 = (
- pose[i, 1] - self.display_radius
- if pose[i, 1] - self.display_radius > 0
- else 0
- )
- y1 = (
- pose[i, 1] + self.display_radius
- if pose[i, 1] + self.display_radius < im_size[0]
- else im_size[0]
- )
- coords = [x0, y0, x1, y1]
- img_draw.ellipse(
- coords,
- fill=self.display_colors[i],
- outline=self.display_colors[i],
- )
- except Exception as e:
- print(e)
-
- imgtk = ImageTk.PhotoImage(image=img)
- self.display_frame_label.imgtk = imgtk
- self.display_frame_label.configure(image=imgtk)
-
- self.display_frame_label.after(10, self.display_frame)
-
- def change_display_keypoints(self):
- """ Toggle display keypoints. If turning on, set display options. If turning off, destroy display window
- """
-
- if self.display_keypoints.get():
-
- display_options = self.cfg["dlc_display_options"][
- self.dlc_option.get()
- ].copy()
- self.display_cmap = display_options["cmap"]
- self.display_radius = display_options["radius"]
- self.display_lik_thresh = display_options["lik_thresh"]
-
- if not self.display_window:
- self.set_display_window()
-
- else:
-
- if self.cam_pose_proc is not None:
- if not self.cam_pose_proc.device.use_tk_display:
- if self.display_window:
- self.display_window.destroy()
- self.display_window = None
- self.display_colors = None
-
- def edit_dlc_display(self):
-
- display_options = self.cfg["dlc_display_options"][self.dlc_option.get()]
-
- dlc_display_settings = {
- "color map": {
- "value": display_options["cmap"],
- "dtype": str,
- "restriction": ["bgy", "kbc", "bmw", "bmy", "kgy", "fire"],
- },
- "radius": {"value": display_options["radius"], "dtype": int},
- "likelihood threshold": {
- "value": display_options["lik_thresh"],
- "dtype": float,
- },
- }
-
- dlc_display_gui = SettingsWindow(
- title="Edit DLC Display Settings",
- settings=dlc_display_settings,
- parent=self.window,
- )
-
- dlc_display_gui.mainloop()
- display_settings = dlc_display_gui.get_values()
-
- display_options["cmap"] = display_settings["color map"]
- display_options["radius"] = display_settings["radius"]
- display_options["lik_thresh"] = display_settings["likelihood threshold"]
-
- self.display_cmap = display_options["cmap"]
- self.display_radius = display_options["radius"]
- self.display_lik_thresh = display_options["lik_thresh"]
-
- self.cfg["dlc_display_options"][self.dlc_option.get()] = display_options
- self.save_config()
-
- def close_camera(self):
- """ Close capture process and display
- """
-
- if self.cam_pose_proc:
- if self.display_window is not None:
- self.display_window.destroy()
- self.display_window = None
- ret = self.cam_pose_proc.stop_capture_process()
-
- self.cam_pose_proc = None
-
- def change_dlc_option(self, event=None):
-
- if self.dlc_option.get() == "Add DLC":
- self.edit_dlc_settings(True)
-
- def edit_dlc_settings(self, new=False):
-
- if new:
- cur_set = self.empty_dlc_settings()
- else:
- cur_set = self.cfg["dlc_options"][self.dlc_option.get()].copy()
- cur_set["name"] = self.dlc_option.get()
- cur_set["cropping"] = (
- ", ".join([str(c) for c in cur_set["cropping"]])
- if cur_set["cropping"]
- else ""
- )
- cur_set["dynamic"] = ", ".join([str(d) for d in cur_set["dynamic"]])
- cur_set["mode"] = (
- "Optimize Latency" if "mode" not in cur_set else cur_set["mode"]
- )
-
- self.dlc_settings_window = Toplevel(self.window)
- self.dlc_settings_window.title("DLC Settings")
- cur_row = 0
-
- Label(self.dlc_settings_window, text="Name: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_name = StringVar(
- self.dlc_settings_window, value=cur_set["name"]
- )
- Entry(self.dlc_settings_window, textvariable=self.dlc_settings_name).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Model Path: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_model_path = StringVar(
- self.dlc_settings_window, value=cur_set["model_path"]
- )
- Entry(self.dlc_settings_window, textvariable=self.dlc_settings_model_path).grid(
- sticky="nsew", row=cur_row, column=1
- )
- Button(
- self.dlc_settings_window, text="Browse", command=self.browse_dlc_path
- ).grid(sticky="nsew", row=cur_row, column=2)
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Model Type: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_model_type = StringVar(
- self.dlc_settings_window, value=cur_set["model_type"]
- )
- Combobox(
- self.dlc_settings_window,
- textvariable=self.dlc_settings_model_type,
- value=["base", "tensorrt", "tflite"],
- state="readonly",
- ).grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Precision: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_precision = StringVar(
- self.dlc_settings_window, value=cur_set["precision"]
- )
- Combobox(
- self.dlc_settings_window,
- textvariable=self.dlc_settings_precision,
- value=["FP32", "FP16", "INT8"],
- state="readonly",
- ).grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Cropping: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_cropping = StringVar(
- self.dlc_settings_window, value=cur_set["cropping"]
- )
- Entry(self.dlc_settings_window, textvariable=self.dlc_settings_cropping).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Dynamic: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_dynamic = StringVar(
- self.dlc_settings_window, value=cur_set["dynamic"]
- )
- Entry(self.dlc_settings_window, textvariable=self.dlc_settings_dynamic).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Resize: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_resize = StringVar(
- self.dlc_settings_window, value=cur_set["resize"]
- )
- Entry(self.dlc_settings_window, textvariable=self.dlc_settings_resize).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 1
-
- Label(self.dlc_settings_window, text="Mode: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_settings_mode = StringVar(
- self.dlc_settings_window, value=cur_set["mode"]
- )
- Combobox(
- self.dlc_settings_window,
- textvariable=self.dlc_settings_mode,
- state="readonly",
- values=["Optimize Latency", "Optimize Rate"],
- ).grid(sticky="nsew", row=cur_row, column=1)
- cur_row += 1
-
- Button(
- self.dlc_settings_window, text="Update", command=self.update_dlc_settings
- ).grid(sticky="nsew", row=cur_row, column=1)
- Button(
- self.dlc_settings_window,
- text="Cancel",
- command=self.dlc_settings_window.destroy,
- ).grid(sticky="nsew", row=cur_row, column=2)
-
- def empty_dlc_settings(self):
-
- return {
- "name": "",
- "model_path": "",
- "model_type": "base",
- "precision": "FP32",
- "cropping": "",
- "dynamic": "False, 0.5, 10",
- "resize": "1.0",
- "mode": "Optimize Latency",
- }
-
- def browse_dlc_path(self):
- """ Open file browser to select DLC exported model directory
- """
-
- new_dlc_path = filedialog.askdirectory(parent=self.dlc_settings_window)
- if new_dlc_path:
- self.dlc_settings_model_path.set(new_dlc_path)
-
- def update_dlc_settings(self):
- """ Update DLC settings for the current dlc option from DLC Settings GUI
- """
-
- precision = (
- self.dlc_settings_precision.get()
- if self.dlc_settings_precision.get()
- else "FP32"
- )
-
- crop_warn = False
- dlc_crop = self.dlc_settings_cropping.get()
- if dlc_crop:
- try:
- dlc_crop = dlc_crop.split(",")
- assert len(dlc_crop) == 4
- dlc_crop = [int(c) for c in dlc_crop]
- except Exception:
- crop_warn = True
- dlc_crop = None
- else:
- dlc_crop = None
-
- try:
- dlc_dynamic = self.dlc_settings_dynamic.get().replace(" ", "")
- dlc_dynamic = dlc_dynamic.split(",")
- dlc_dynamic[0] = True if dlc_dynamic[0] == "True" else False
- dlc_dynamic[1] = float(dlc_dynamic[1])
- dlc_dynamic[2] = int(dlc_dynamic[2])
- dlc_dynamic = tuple(dlc_dynamic)
- dyn_warn = False
- except Exception:
- dyn_warn = True
- dlc_dynamic = (False, 0.5, 10)
-
- dlc_resize = (
- float(self.dlc_settings_resize.get())
- if self.dlc_settings_resize.get()
- else None
- )
- dlc_mode = self.dlc_settings_mode.get()
-
- warn_msg = ""
- if crop_warn:
- warn_msg += "DLC Cropping was not set properly. Using default cropping parameters...\n"
- if dyn_warn:
- warn_msg += "DLC Dynamic Cropping was not set properly. Using default dynamic cropping parameters..."
- if warn_msg:
- messagebox.showerror(
- "DLC Settings Error", warn_msg, parent=self.dlc_settings_window
- )
-
- self.cfg["dlc_options"][self.dlc_settings_name.get()] = {
- "model_path": self.dlc_settings_model_path.get(),
- "model_type": self.dlc_settings_model_type.get(),
- "precision": precision,
- "cropping": dlc_crop,
- "dynamic": dlc_dynamic,
- "resize": dlc_resize,
- "mode": dlc_mode,
- }
-
- if self.dlc_settings_name.get() not in self.cfg["dlc_display_options"]:
- self.cfg["dlc_display_options"][self.dlc_settings_name.get()] = {
- "cmap": "bgy",
- "radius": 3,
- "lik_thresh": 0.5,
- }
-
- self.save_config()
- self.dlc_options_entry["values"] = tuple(self.cfg["dlc_options"].keys()) + (
- "Add DLC",
- )
- self.dlc_option.set(self.dlc_settings_name.get())
- self.dlc_settings_window.destroy()
-
- def remove_dlc_option(self):
- """ Delete DLC Option from config
- """
-
- del self.cfg["dlc_options"][self.dlc_option.get()]
- del self.cfg["dlc_display_options"][self.dlc_option.get()]
- self.dlc_options_entry["values"] = tuple(self.cfg["dlc_options"].keys()) + (
- "Add DLC",
- )
- self.dlc_option.set("")
- self.save_config()
-
- def init_dlc(self):
- """ Initialize DLC Live object
- """
-
- self.stop_pose()
-
- self.dlc_setup_window = Toplevel(self.window)
- self.dlc_setup_window.title("Setting up DLC...")
- Label(self.dlc_setup_window, text="Setting up DLC, please wait...").pack()
- self.dlc_setup_window.after(10, self.start_pose)
- self.dlc_setup_window.mainloop()
-
- def start_pose(self):
-
- dlc_params = self.cfg["dlc_options"][self.dlc_option.get()].copy()
- dlc_params["processor"] = self.dlc_proc_params
- ret = self.cam_pose_proc.start_pose_process(dlc_params)
- self.dlc_setup_window.destroy()
-
- def stop_pose(self):
- """ Stop pose process
- """
-
- if self.cam_pose_proc:
- ret = self.cam_pose_proc.stop_pose_process()
-
- def add_subject(self):
- new_sub = self.subject.get()
- if new_sub:
- if new_sub not in self.cfg["subjects"]:
- self.cfg["subjects"].append(new_sub)
- self.subject_entry["values"] = tuple(self.cfg["subjects"])
- self.save_config()
-
- def remove_subject(self):
-
- self.cfg["subjects"].remove(self.subject.get())
- self.subject_entry["values"] = self.cfg["subjects"]
- self.save_config()
- self.subject.set("")
-
- def browse_directory(self):
-
- new_dir = filedialog.askdirectory(parent=self.window)
- if new_dir:
- self.directory.set(new_dir)
- ask_add_dir = Tk()
- Label(
- ask_add_dir,
- text="Would you like to add this directory to dropdown list?",
- ).pack()
- Button(
- ask_add_dir, text="Yes", command=lambda: self.add_directory(ask_add_dir)
- ).pack()
- Button(ask_add_dir, text="No", command=ask_add_dir.destroy).pack()
-
- def add_directory(self, window):
-
- window.destroy()
- if self.directory.get() not in self.cfg["directories"]:
- self.cfg["directories"].append(self.directory.get())
- self.directory_entry["values"] = self.cfg["directories"]
- self.save_config()
-
- def save_config(self, notify=False):
-
- json.dump(self.cfg, open(self.cfg_file, "w"))
- if notify:
- messagebox.showinfo(
- title="Config file saved",
- message="Configuration file has been saved...",
- parent=self.window,
- )
-
- def remove_cam_cfg(self):
-
- if self.camera_name.get() != "Add Camera":
- delete = messagebox.askyesno(
- title="Delete Camera?",
- message="Are you sure you want to delete '%s'?"
- % self.camera_name.get(),
- parent=self.window,
- )
- if delete:
- del self.cfg["cameras"][self.camera_name.get()]
- self.camera_entry["values"] = self.get_camera_names() + ("Add Camera",)
- self.camera_name.set("")
- self.save_config()
-
- def browse_dlc_processor(self):
-
- new_dir = filedialog.askdirectory(parent=self.window)
- if new_dir:
- self.dlc_proc_dir.set(new_dir)
- self.update_dlc_proc_list()
-
- if new_dir not in self.cfg["processor_dir"]:
- if messagebox.askyesno(
- "Add to dropdown",
- "Would you like to add this directory to dropdown list?",
- ):
- self.cfg["processor_dir"].append(new_dir)
- self.dlc_proc_dir_entry["values"] = tuple(self.cfg["processor_dir"])
- self.save_config()
-
- def rem_dlc_proc_dir(self):
-
- if self.dlc_proc_dir.get() in self.cfg["processor_dir"]:
- self.cfg["processor_dir"].remove(self.dlc_proc_dir.get())
- self.save_config()
- self.dlc_proc_dir_entry["values"] = tuple(self.cfg["processor_dir"])
- self.dlc_proc_dir.set("")
-
- def update_dlc_proc_list(self, event=None):
-
- ### if dlc proc module already initialized, delete module and remove from path ###
-
- self.processor_list = []
-
- if self.dlc_proc_dir.get():
-
- if hasattr(self, "dlc_proc_module"):
- sys.path.remove(sys.path[0])
-
- new_path = os.path.normpath(os.path.dirname(self.dlc_proc_dir.get()))
- if new_path not in sys.path:
- sys.path.insert(0, new_path)
-
- new_mod = os.path.basename(self.dlc_proc_dir.get())
- if new_mod in sys.modules:
- del sys.modules[new_mod]
-
- ### load new module ###
-
- processor_spec = importlib.util.find_spec(
- os.path.basename(self.dlc_proc_dir.get())
- )
- try:
- self.dlc_proc_module = importlib.util.module_from_spec(processor_spec)
- processor_spec.loader.exec_module(self.dlc_proc_module)
- # self.processor_list = inspect.getmembers(self.dlc_proc_module, inspect.isclass)
- self.processor_list = [
- proc for proc in dir(self.dlc_proc_module) if "__" not in proc
- ]
- except AttributeError:
- if hasattr(self, "window"):
- messagebox.showerror(
- "Failed to load processors!",
- "Failed to load processors from directory = "
- + self.dlc_proc_dir.get()
- + ".\nPlease select a different directory.",
- parent=self.window,
- )
-
- self.dlc_proc_name_entry["values"] = tuple(self.processor_list)
-
- def set_proc(self):
-
- # proc_param_dict = {}
- # for i in range(1, len(self.proc_param_names)):
- # proc_param_dict[self.proc_param_names[i]] = self.proc_param_default_types[i](self.proc_param_values[i-1].get())
-
- # if self.dlc_proc_dir.get() not in self.cfg['processor_args']:
- # self.cfg['processor_args'][self.dlc_proc_dir.get()] = {}
- # self.cfg['processor_args'][self.dlc_proc_dir.get()][self.dlc_proc_name.get()] = proc_param_dict
- # self.save_config()
-
- # self.dlc_proc = self.proc_object(**proc_param_dict)
- proc_object = getattr(self.dlc_proc_module, self.dlc_proc_name.get())
- self.dlc_proc_params = {"object": proc_object}
- self.dlc_proc_params.update(
- self.cfg["processor_args"][self.dlc_proc_dir.get()][
- self.dlc_proc_name.get()
- ]
- )
-
- def clear_proc(self):
-
- self.dlc_proc_params = None
-
- def edit_proc(self):
-
- ### get default args: load module and read arguments ###
-
- self.proc_object = getattr(self.dlc_proc_module, self.dlc_proc_name.get())
- def_args = inspect.getargspec(self.proc_object)
- self.proc_param_names = def_args[0]
- self.proc_param_default_values = def_args[3]
- self.proc_param_default_types = [
- type(v) if type(v) is not list else [type(v[0])] for v in def_args[3]
- ]
- for i in range(len(def_args[0]) - len(def_args[3])):
- self.proc_param_default_values = ("",) + self.proc_param_default_values
- self.proc_param_default_types = [str] + self.proc_param_default_types
-
- ### check for existing settings in config ###
-
- old_args = {}
- if self.dlc_proc_dir.get() in self.cfg["processor_args"]:
- if (
- self.dlc_proc_name.get()
- in self.cfg["processor_args"][self.dlc_proc_dir.get()]
- ):
- old_args = self.cfg["processor_args"][self.dlc_proc_dir.get()][
- self.dlc_proc_name.get()
- ].copy()
- else:
- self.cfg["processor_args"][self.dlc_proc_dir.get()] = {}
-
- ### get dictionary of arguments ###
-
- proc_args_dict = {}
- for i in range(1, len(self.proc_param_names)):
-
- if self.proc_param_names[i] in old_args:
- this_value = old_args[self.proc_param_names[i]]
- else:
- this_value = self.proc_param_default_values[i]
-
- proc_args_dict[self.proc_param_names[i]] = {
- "value": this_value,
- "dtype": self.proc_param_default_types[i],
- }
-
- proc_args_gui = SettingsWindow(
- title="DLC Processor Settings", settings=proc_args_dict, parent=self.window
- )
- proc_args_gui.mainloop()
-
- self.cfg["processor_args"][self.dlc_proc_dir.get()][
- self.dlc_proc_name.get()
- ] = proc_args_gui.get_values()
- self.save_config()
-
- def init_session(self):
-
- ### check if video is currently open ###
-
- if self.record_on.get() > -1:
- messagebox.showerror(
- "Session Open",
- "Session is currently open! Please release the current video (click 'Save Video' of 'Delete Video', even if no frames have been recorded) before setting up a new one.",
- parent=self.window,
- )
- return
-
- ### check if camera is already set up ###
-
- if not self.cam_pose_proc:
- messagebox.showerror(
- "No Camera",
- "No camera is found! Please initialize a camera before setting up the video.",
- parent=self.window,
- )
- return
-
- ### set up session window
-
- self.session_setup_window = Toplevel(self.window)
- self.session_setup_window.title("Setting up session...")
- Label(
- self.session_setup_window, text="Setting up session, please wait..."
- ).pack()
- self.session_setup_window.after(10, self.start_writer)
- self.session_setup_window.mainloop()
-
- def start_writer(self):
-
- ### set up file name (get date and create directory)
-
- dt = datetime.datetime.now()
- date = f"{dt.year:04d}-{dt.month:02d}-{dt.day:02d}"
- self.out_dir = self.directory.get()
- if not os.path.isdir(os.path.normpath(self.out_dir)):
- os.makedirs(os.path.normpath(self.out_dir))
-
- ### create output file names
-
- self.base_name = os.path.normpath(
- f"{self.out_dir}/{self.camera_name.get().replace(' ', '')}_{self.subject.get()}_{date}_{self.attempt.get()}"
- )
- # self.vid_file = os.path.normpath(self.out_dir + '/VIDEO_' + self.base_name + '.avi')
- # self.ts_file = os.path.normpath(self.out_dir + '/TIMESTAMPS_' + self.base_name + '.pickle')
- # self.dlc_file = os.path.normpath(self.out_dir + '/DLC_' + self.base_name + '.h5')
- # self.proc_file = os.path.normpath(self.out_dir + '/PROC_' + self.base_name + '.pickle')
-
- ### check if files already exist
-
- fs = glob.glob(f"{self.base_name}*")
- if len(fs) > 0:
- overwrite = messagebox.askyesno(
- "Files Exist",
- "Files already exist with attempt number = {}. Would you like to overwrite the file?".format(
- self.attempt.get()
- ),
- parent=self.session_setup_window,
- )
- if not overwrite:
- return
-
- ### start writer
-
- ret = self.cam_pose_proc.start_writer_process(self.base_name)
-
- self.session_setup_window.destroy()
-
- ### set GUI to Ready
-
- self.record_on.set(0)
-
- def start_record(self):
- """ Issues command to start recording frames and poses
- """
-
- ret = False
- if self.cam_pose_proc is not None:
- ret = self.cam_pose_proc.start_record()
-
- if not ret:
- messagebox.showerror(
- "Recording Not Ready",
- "Recording has not been set up. Please make sure a camera and session have been initialized.",
- parent=self.window,
- )
- self.record_on.set(-1)
-
- def stop_record(self):
- """ Issues command to stop recording frames and poses
- """
-
- if self.cam_pose_proc is not None:
- ret = self.cam_pose_proc.stop_record()
- self.record_on.set(0)
-
- def save_vid(self, delete=False):
- """ Saves video, timestamp, and DLC files
-
- Parameters
- ----------
- delete : bool, optional
- flag to delete created files, by default False
- """
-
- ### perform checks ###
-
- if self.cam_pose_proc is None:
- messagebox.showwarning(
- "No Camera",
- "Camera has not yet been initialized, no video recorded.",
- parent=self.window,
- )
- return
-
- elif self.record_on.get() == -1:
- messagebox.showwarning(
- "No Video File",
- "Video was not set up, no video recorded.",
- parent=self.window,
- )
- return
-
- elif self.record_on.get() == 1:
- messagebox.showwarning(
- "Active Recording",
- "You are currently recording a video. Please stop the video before saving.",
- parent=self.window,
- )
- return
-
- elif delete:
- delete = messagebox.askokcancel(
- "Delete Video?", "Do you wish to delete the video?", parent=self.window
- )
-
- ### save or delete video ###
-
- if delete:
- ret = self.cam_pose_proc.stop_writer_process(save=False)
- messagebox.showinfo(
- "Video Deleted",
- "Video and timestamp files have been deleted.",
- parent=self.window,
- )
- else:
- ret = self.cam_pose_proc.stop_writer_process(save=True)
- ret_pose = self.cam_pose_proc.save_pose(self.base_name)
- if ret:
- if ret_pose:
- messagebox.showinfo(
- "Files Saved",
- "Video, timestamp, and DLC Files have been saved.",
- )
- else:
- messagebox.showinfo(
- "Files Saved", "Video and timestamp files have been saved."
- )
- else:
- messagebox.showwarning(
- "No Frames Recorded",
- "No frames were recorded, video was deleted",
- parent=self.window,
- )
-
- self.record_on.set(-1)
-
- def closeGUI(self):
-
- if self.cam_pose_proc:
- ret = self.cam_pose_proc.stop_writer_process()
- ret = self.cam_pose_proc.stop_pose_process()
- ret = self.cam_pose_proc.stop_capture_process()
-
- self.window.destroy()
-
- def createGUI(self):
-
- ### initialize window ###
-
- self.window = Tk()
- self.window.title("DeepLabCut Live")
- cur_row = 0
- combobox_width = 15
-
- ### select cfg file
- if len(self.cfg_list) > 0:
- initial_cfg = self.cfg_list[0]
- else:
- initial_cfg = ""
-
- Label(self.window, text="Config: ").grid(sticky="w", row=cur_row, column=0)
- self.cfg_name = StringVar(self.window, value=initial_cfg)
- self.cfg_entry = Combobox(
- self.window, textvariable=self.cfg_name, width=combobox_width
- )
- self.cfg_entry["values"] = tuple(self.cfg_list) + ("Create New Config",)
- self.cfg_entry.bind("<>", self.change_config)
- self.cfg_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Remove Config", command=self.remove_config).grid(
- sticky="nsew", row=cur_row, column=2
- )
-
- self.get_config(initial_cfg)
-
- cur_row += 2
-
- ### select camera ###
-
- # camera entry
- Label(self.window, text="Camera: ").grid(sticky="w", row=cur_row, column=0)
- self.camera_name = StringVar(self.window)
- self.camera_entry = Combobox(self.window, textvariable=self.camera_name)
- cam_names = self.get_camera_names()
- self.camera_entry["values"] = cam_names + ("Add Camera",)
- if cam_names:
- self.camera_entry.current(0)
- self.camera_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Init Cam", command=self.init_cam).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- Button(
- self.window, text="Edit Camera Settings", command=self.edit_cam_settings
- ).grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Close Camera", command=self.close_camera).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
- Button(self.window, text="Remove Camera", command=self.remove_cam_cfg).grid(
- sticky="nsew", row=cur_row, column=2
- )
-
- cur_row += 2
-
- ### set up proc ###
-
- Label(self.window, text="Processor Dir: ").grid(
- sticky="w", row=cur_row, column=0
- )
- self.dlc_proc_dir = StringVar(self.window)
- self.dlc_proc_dir_entry = Combobox(self.window, textvariable=self.dlc_proc_dir)
- self.dlc_proc_dir_entry["values"] = tuple(self.cfg["processor_dir"])
- if len(self.cfg["processor_dir"]) > 0:
- self.dlc_proc_dir_entry.current(0)
- self.dlc_proc_dir_entry.bind("<>", self.update_dlc_proc_list)
- self.dlc_proc_dir_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Browse", command=self.browse_dlc_processor).grid(
- sticky="nsew", row=cur_row, column=2
- )
- Button(self.window, text="Remove Proc Dir", command=self.rem_dlc_proc_dir).grid(
- sticky="nsew", row=cur_row + 1, column=2
- )
- cur_row += 2
-
- Label(self.window, text="Processor: ").grid(sticky="w", row=cur_row, column=0)
- self.dlc_proc_name = StringVar(self.window)
- self.dlc_proc_name_entry = Combobox(
- self.window, textvariable=self.dlc_proc_name
- )
- self.update_dlc_proc_list()
- # self.dlc_proc_name_entry['values'] = tuple(self.processor_list) # tuple([c[0] for c in inspect.getmembers(processor, inspect.isclass)])
- if len(self.processor_list) > 0:
- self.dlc_proc_name_entry.current(0)
- self.dlc_proc_name_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Set Proc", command=self.set_proc).grid(
- sticky="nsew", row=cur_row, column=2
- )
- Button(self.window, text="Edit Proc Settings", command=self.edit_proc).grid(
- sticky="nsew", row=cur_row + 1, column=1
- )
- Button(self.window, text="Clear Proc", command=self.clear_proc).grid(
- sticky="nsew", row=cur_row + 1, column=2
- )
-
- cur_row += 3
-
- ### set up dlc live ###
-
- Label(self.window, text="DeepLabCut: ").grid(sticky="w", row=cur_row, column=0)
- self.dlc_option = StringVar(self.window)
- self.dlc_options_entry = Combobox(self.window, textvariable=self.dlc_option)
- self.dlc_options_entry["values"] = tuple(self.cfg["dlc_options"].keys()) + (
- "Add DLC",
- )
- self.dlc_options_entry.bind("<>", self.change_dlc_option)
- self.dlc_options_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Init DLC", command=self.init_dlc).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- Button(
- self.window, text="Edit DLC Settings", command=self.edit_dlc_settings
- ).grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Stop DLC", command=self.stop_pose).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- self.display_keypoints = BooleanVar(self.window, value=False)
- Checkbutton(
- self.window,
- text="Display DLC Keypoints",
- variable=self.display_keypoints,
- indicatoron=0,
- command=self.change_display_keypoints,
- ).grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Remove DLC", command=self.remove_dlc_option).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- Button(
- self.window, text="Edit DLC Display Settings", command=self.edit_dlc_display
- ).grid(sticky="nsew", row=cur_row, column=1)
-
- cur_row += 2
-
- ### set up session ###
-
- # subject
- Label(self.window, text="Subject: ").grid(sticky="w", row=cur_row, column=0)
- self.subject = StringVar(self.window)
- self.subject_entry = Combobox(self.window, textvariable=self.subject)
- self.subject_entry["values"] = self.cfg["subjects"]
- self.subject_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Add Subject", command=self.add_subject).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- # attempt
- Label(self.window, text="Attempt: ").grid(sticky="w", row=cur_row, column=0)
- self.attempt = StringVar(self.window)
- self.attempt_entry = Combobox(self.window, textvariable=self.attempt)
- self.attempt_entry["values"] = tuple(range(1, 10))
- self.attempt_entry.current(0)
- self.attempt_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Remove Subject", command=self.remove_subject).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- # out directory
- Label(self.window, text="Directory: ").grid(sticky="w", row=cur_row, column=0)
- self.directory = StringVar(self.window)
- self.directory_entry = Combobox(self.window, textvariable=self.directory)
- if self.cfg["directories"]:
- self.directory_entry["values"] = self.cfg["directories"]
- self.directory_entry.current(0)
- self.directory_entry.grid(sticky="nsew", row=cur_row, column=1)
- Button(self.window, text="Browse", command=self.browse_directory).grid(
- sticky="nsew", row=cur_row, column=2
- )
- cur_row += 1
-
- # set up session
- Button(self.window, text="Set Up Session", command=self.init_session).grid(
- sticky="nsew", row=cur_row, column=1
- )
- cur_row += 2
-
- ### control recording ###
-
- Label(self.window, text="Record: ").grid(sticky="w", row=cur_row, column=0)
- self.record_on = IntVar(value=-1)
- Radiobutton(
- self.window,
- text="Ready",
- selectcolor="blue",
- indicatoron=0,
- variable=self.record_on,
- value=0,
- state="disabled",
- ).grid(stick="nsew", row=cur_row, column=1)
- Radiobutton(
- self.window,
- text="On",
- selectcolor="green",
- indicatoron=0,
- variable=self.record_on,
- value=1,
- command=self.start_record,
- ).grid(sticky="nsew", row=cur_row + 1, column=1)
- Radiobutton(
- self.window,
- text="Off",
- selectcolor="red",
- indicatoron=0,
- variable=self.record_on,
- value=-1,
- command=self.stop_record,
- ).grid(sticky="nsew", row=cur_row + 2, column=1)
- Button(self.window, text="Save Video", command=lambda: self.save_vid()).grid(
- sticky="nsew", row=cur_row + 1, column=2
- )
- Button(
- self.window, text="Delete Video", command=lambda: self.save_vid(delete=True)
- ).grid(sticky="nsew", row=cur_row + 2, column=2)
-
- cur_row += 4
-
- ### close program ###
-
- Button(self.window, text="Close", command=self.closeGUI).grid(
- sticky="nsew", row=cur_row, column=0, columnspan=2
- )
-
- ### configure size of empty rows
-
- _, row_count = self.window.grid_size()
- for r in range(row_count):
- self.window.grid_rowconfigure(r, minsize=20)
-
- def run(self):
-
- self.window.mainloop()
-
-
-def main():
-
- # import multiprocess as mp
- # mp.set_start_method("spawn")
-
- dlc_live_gui = DLCLiveGUI()
- dlc_live_gui.run()
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
new file mode 100644
index 0000000..4eb3971
--- /dev/null
+++ b/dlclivegui/gui.py
@@ -0,0 +1,542 @@
+"""PyQt6 based GUI for DeepLabCut Live."""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Optional
+
+import cv2
+import numpy as np
+from PyQt6.QtCore import Qt
+from PyQt6.QtGui import QAction, QCloseEvent, QImage, QPixmap
+from PyQt6.QtWidgets import (
+ QApplication,
+ QCheckBox,
+ QComboBox,
+ QFileDialog,
+ QFormLayout,
+ QGroupBox,
+ QHBoxLayout,
+ QLabel,
+ QLineEdit,
+ QMainWindow,
+ QMessageBox,
+ QPlainTextEdit,
+ QPushButton,
+ QSpinBox,
+ QDoubleSpinBox,
+ QStatusBar,
+ QVBoxLayout,
+ QWidget,
+)
+
+from .camera_controller import CameraController, FrameData
+from .cameras import CameraFactory
+from .config import (
+ ApplicationSettings,
+ CameraSettings,
+ DLCProcessorSettings,
+ RecordingSettings,
+ DEFAULT_CONFIG,
+)
+from .dlc_processor import DLCLiveProcessor, PoseResult
+from .video_recorder import VideoRecorder
+
+
+class MainWindow(QMainWindow):
+ """Main application window."""
+
+ def __init__(self, config: Optional[ApplicationSettings] = None):
+ super().__init__()
+ self.setWindowTitle("DeepLabCut Live GUI")
+ self._config = config or DEFAULT_CONFIG
+ self._config_path: Optional[Path] = None
+ self._current_frame: Optional[np.ndarray] = None
+ self._last_pose: Optional[PoseResult] = None
+ self._video_recorder: Optional[VideoRecorder] = None
+
+ self.camera_controller = CameraController()
+ self.dlc_processor = DLCLiveProcessor()
+
+ self._setup_ui()
+ self._connect_signals()
+ self._apply_config(self._config)
+
+ # ------------------------------------------------------------------ UI
+ def _setup_ui(self) -> None:
+ central = QWidget()
+ layout = QVBoxLayout(central)
+
+ self.video_label = QLabel("Camera preview not started")
+ self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
+ self.video_label.setMinimumSize(640, 360)
+ layout.addWidget(self.video_label)
+
+ layout.addWidget(self._build_camera_group())
+ layout.addWidget(self._build_dlc_group())
+ layout.addWidget(self._build_recording_group())
+
+ button_bar = QHBoxLayout()
+ self.preview_button = QPushButton("Start Preview")
+ self.stop_preview_button = QPushButton("Stop Preview")
+ self.stop_preview_button.setEnabled(False)
+ button_bar.addWidget(self.preview_button)
+ button_bar.addWidget(self.stop_preview_button)
+ layout.addLayout(button_bar)
+
+ self.setCentralWidget(central)
+ self.setStatusBar(QStatusBar())
+ self._build_menus()
+
+ def _build_menus(self) -> None:
+ file_menu = self.menuBar().addMenu("&File")
+
+ load_action = QAction("Load configuration…", self)
+ load_action.triggered.connect(self._action_load_config)
+ file_menu.addAction(load_action)
+
+ save_action = QAction("Save configuration", self)
+ save_action.triggered.connect(self._action_save_config)
+ file_menu.addAction(save_action)
+
+ save_as_action = QAction("Save configuration as…", self)
+ save_as_action.triggered.connect(self._action_save_config_as)
+ file_menu.addAction(save_as_action)
+
+ file_menu.addSeparator()
+ exit_action = QAction("Exit", self)
+ exit_action.triggered.connect(self.close)
+ file_menu.addAction(exit_action)
+
+ def _build_camera_group(self) -> QGroupBox:
+ group = QGroupBox("Camera settings")
+ form = QFormLayout(group)
+
+ self.camera_index = QComboBox()
+ self.camera_index.setEditable(True)
+ self.camera_index.addItems([str(i) for i in range(5)])
+ form.addRow("Camera index", self.camera_index)
+
+ self.camera_width = QSpinBox()
+ self.camera_width.setRange(1, 7680)
+ form.addRow("Width", self.camera_width)
+
+ self.camera_height = QSpinBox()
+ self.camera_height.setRange(1, 4320)
+ form.addRow("Height", self.camera_height)
+
+ self.camera_fps = QDoubleSpinBox()
+ self.camera_fps.setRange(1.0, 240.0)
+ self.camera_fps.setDecimals(2)
+ form.addRow("Frame rate", self.camera_fps)
+
+ self.camera_backend = QComboBox()
+ self.camera_backend.setEditable(True)
+ availability = CameraFactory.available_backends()
+ for backend in CameraFactory.backend_names():
+ label = backend
+ if not availability.get(backend, True):
+ label = f"{backend} (unavailable)"
+ self.camera_backend.addItem(label, backend)
+ form.addRow("Backend", self.camera_backend)
+
+ self.camera_properties_edit = QPlainTextEdit()
+ self.camera_properties_edit.setPlaceholderText(
+ '{"exposure": 15000, "gain": 0.5, "serial": "123456"}'
+ )
+ self.camera_properties_edit.setFixedHeight(60)
+ form.addRow("Advanced properties", self.camera_properties_edit)
+
+ return group
+
+ def _build_dlc_group(self) -> QGroupBox:
+ group = QGroupBox("DLCLive settings")
+ form = QFormLayout(group)
+
+ path_layout = QHBoxLayout()
+ self.model_path_edit = QLineEdit()
+ path_layout.addWidget(self.model_path_edit)
+ browse_model = QPushButton("Browse…")
+ browse_model.clicked.connect(self._action_browse_model)
+ path_layout.addWidget(browse_model)
+ form.addRow("Model path", path_layout)
+
+ self.shuffle_edit = QLineEdit()
+ self.shuffle_edit.setPlaceholderText("Optional integer")
+ form.addRow("Shuffle", self.shuffle_edit)
+
+ self.training_edit = QLineEdit()
+ self.training_edit.setPlaceholderText("Optional integer")
+ form.addRow("Training set index", self.training_edit)
+
+ self.processor_combo = QComboBox()
+ self.processor_combo.setEditable(True)
+ self.processor_combo.addItems(["cpu", "gpu", "tensorrt"])
+ form.addRow("Processor", self.processor_combo)
+
+ self.processor_args_edit = QPlainTextEdit()
+ self.processor_args_edit.setPlaceholderText('{"device": 0}')
+ self.processor_args_edit.setFixedHeight(60)
+ form.addRow("Processor args", self.processor_args_edit)
+
+ self.additional_options_edit = QPlainTextEdit()
+ self.additional_options_edit.setPlaceholderText('{"allow_growth": true}')
+ self.additional_options_edit.setFixedHeight(60)
+ form.addRow("Additional options", self.additional_options_edit)
+
+ self.enable_dlc_checkbox = QCheckBox("Enable pose estimation")
+ self.enable_dlc_checkbox.setChecked(True)
+ form.addRow(self.enable_dlc_checkbox)
+
+ return group
+
+ def _build_recording_group(self) -> QGroupBox:
+ group = QGroupBox("Recording")
+ form = QFormLayout(group)
+
+ self.recording_enabled_checkbox = QCheckBox("Record video while running")
+ form.addRow(self.recording_enabled_checkbox)
+
+ dir_layout = QHBoxLayout()
+ self.output_directory_edit = QLineEdit()
+ dir_layout.addWidget(self.output_directory_edit)
+ browse_dir = QPushButton("Browse…")
+ browse_dir.clicked.connect(self._action_browse_directory)
+ dir_layout.addWidget(browse_dir)
+ form.addRow("Output directory", dir_layout)
+
+ self.filename_edit = QLineEdit()
+ form.addRow("Filename", self.filename_edit)
+
+ self.container_combo = QComboBox()
+ self.container_combo.setEditable(True)
+ self.container_combo.addItems(["mp4", "avi", "mov"])
+ form.addRow("Container", self.container_combo)
+
+ self.recording_options_edit = QPlainTextEdit()
+ self.recording_options_edit.setPlaceholderText('{"compression_mode": "mp4"}')
+ self.recording_options_edit.setFixedHeight(60)
+ form.addRow("WriteGear options", self.recording_options_edit)
+
+ self.start_record_button = QPushButton("Start recording")
+ self.stop_record_button = QPushButton("Stop recording")
+ self.stop_record_button.setEnabled(False)
+
+ buttons = QHBoxLayout()
+ buttons.addWidget(self.start_record_button)
+ buttons.addWidget(self.stop_record_button)
+ form.addRow(buttons)
+
+ return group
+
+ # ------------------------------------------------------------------ signals
+ def _connect_signals(self) -> None:
+ self.preview_button.clicked.connect(self._start_preview)
+ self.stop_preview_button.clicked.connect(self._stop_preview)
+ self.start_record_button.clicked.connect(self._start_recording)
+ self.stop_record_button.clicked.connect(self._stop_recording)
+
+ self.camera_controller.frame_ready.connect(self._on_frame_ready)
+ self.camera_controller.error.connect(self._show_error)
+ self.camera_controller.stopped.connect(self._on_camera_stopped)
+
+ self.dlc_processor.pose_ready.connect(self._on_pose_ready)
+ self.dlc_processor.error.connect(self._show_error)
+ self.dlc_processor.initialized.connect(self._on_dlc_initialised)
+
+ # ------------------------------------------------------------------ config
+ def _apply_config(self, config: ApplicationSettings) -> None:
+ camera = config.camera
+ self.camera_index.setCurrentText(str(camera.index))
+ self.camera_width.setValue(int(camera.width))
+ self.camera_height.setValue(int(camera.height))
+ self.camera_fps.setValue(float(camera.fps))
+ backend_name = camera.backend or "opencv"
+ index = self.camera_backend.findData(backend_name)
+ if index >= 0:
+ self.camera_backend.setCurrentIndex(index)
+ else:
+ self.camera_backend.setEditText(backend_name)
+ self.camera_properties_edit.setPlainText(
+ json.dumps(camera.properties, indent=2) if camera.properties else ""
+ )
+
+ dlc = config.dlc
+ self.model_path_edit.setText(dlc.model_path)
+ self.shuffle_edit.setText("" if dlc.shuffle is None else str(dlc.shuffle))
+ self.training_edit.setText(
+ "" if dlc.trainingsetindex is None else str(dlc.trainingsetindex)
+ )
+ self.processor_combo.setCurrentText(dlc.processor or "cpu")
+ self.processor_args_edit.setPlainText(json.dumps(dlc.processor_args, indent=2))
+ self.additional_options_edit.setPlainText(
+ json.dumps(dlc.additional_options, indent=2)
+ )
+
+ recording = config.recording
+ self.recording_enabled_checkbox.setChecked(recording.enabled)
+ self.output_directory_edit.setText(recording.directory)
+ self.filename_edit.setText(recording.filename)
+ self.container_combo.setCurrentText(recording.container)
+ self.recording_options_edit.setPlainText(json.dumps(recording.options, indent=2))
+
+ def _current_config(self) -> ApplicationSettings:
+ return ApplicationSettings(
+ camera=self._camera_settings_from_ui(),
+ dlc=self._dlc_settings_from_ui(),
+ recording=self._recording_settings_from_ui(),
+ )
+
+ def _camera_settings_from_ui(self) -> CameraSettings:
+ index_text = self.camera_index.currentText().strip() or "0"
+ try:
+ index = int(index_text)
+ except ValueError:
+ raise ValueError("Camera index must be an integer") from None
+ backend_data = self.camera_backend.currentData()
+ backend_text = (
+ backend_data
+ if isinstance(backend_data, str) and backend_data
+ else self.camera_backend.currentText().strip()
+ )
+ properties = self._parse_json(self.camera_properties_edit.toPlainText())
+ return CameraSettings(
+ name=f"Camera {index}",
+ index=index,
+ width=self.camera_width.value(),
+ height=self.camera_height.value(),
+ fps=self.camera_fps.value(),
+ backend=backend_text or "opencv",
+ properties=properties,
+ )
+
+ def _parse_optional_int(self, value: str) -> Optional[int]:
+ text = value.strip()
+ if not text:
+ return None
+ return int(text)
+
+ def _parse_json(self, value: str) -> dict:
+ text = value.strip()
+ if not text:
+ return {}
+ return json.loads(text)
+
+ def _dlc_settings_from_ui(self) -> DLCProcessorSettings:
+ return DLCProcessorSettings(
+ model_path=self.model_path_edit.text().strip(),
+ shuffle=self._parse_optional_int(self.shuffle_edit.text()),
+ trainingsetindex=self._parse_optional_int(self.training_edit.text()),
+ processor=self.processor_combo.currentText().strip() or "cpu",
+ processor_args=self._parse_json(self.processor_args_edit.toPlainText()),
+ additional_options=self._parse_json(
+ self.additional_options_edit.toPlainText()
+ ),
+ )
+
+ def _recording_settings_from_ui(self) -> RecordingSettings:
+ return RecordingSettings(
+ enabled=self.recording_enabled_checkbox.isChecked(),
+ directory=self.output_directory_edit.text().strip(),
+ filename=self.filename_edit.text().strip() or "session.mp4",
+ container=self.container_combo.currentText().strip() or "mp4",
+ options=self._parse_json(self.recording_options_edit.toPlainText()),
+ )
+
+ # ------------------------------------------------------------------ actions
+ def _action_load_config(self) -> None:
+ file_name, _ = QFileDialog.getOpenFileName(
+ self, "Load configuration", str(Path.home()), "JSON files (*.json)"
+ )
+ if not file_name:
+ return
+ try:
+ config = ApplicationSettings.load(file_name)
+ except Exception as exc: # pragma: no cover - GUI interaction
+ self._show_error(str(exc))
+ return
+ self._config = config
+ self._config_path = Path(file_name)
+ self._apply_config(config)
+ self.statusBar().showMessage(f"Loaded configuration: {file_name}", 5000)
+
+ def _action_save_config(self) -> None:
+ if self._config_path is None:
+ self._action_save_config_as()
+ return
+ self._save_config_to_path(self._config_path)
+
+ def _action_save_config_as(self) -> None:
+ file_name, _ = QFileDialog.getSaveFileName(
+ self, "Save configuration", str(Path.home()), "JSON files (*.json)"
+ )
+ if not file_name:
+ return
+ path = Path(file_name)
+ if path.suffix.lower() != ".json":
+ path = path.with_suffix(".json")
+ self._config_path = path
+ self._save_config_to_path(path)
+
+ def _save_config_to_path(self, path: Path) -> None:
+ try:
+ config = self._current_config()
+ config.save(path)
+ except Exception as exc: # pragma: no cover - GUI interaction
+ self._show_error(str(exc))
+ return
+ self.statusBar().showMessage(f"Saved configuration to {path}", 5000)
+
+ def _action_browse_model(self) -> None:
+ file_name, _ = QFileDialog.getOpenFileName(
+ self, "Select DLCLive model", str(Path.home()), "All files (*.*)"
+ )
+ if file_name:
+ self.model_path_edit.setText(file_name)
+
+ def _action_browse_directory(self) -> None:
+ directory = QFileDialog.getExistingDirectory(
+ self, "Select output directory", str(Path.home())
+ )
+ if directory:
+ self.output_directory_edit.setText(directory)
+
+ # ------------------------------------------------------------------ camera control
+ def _start_preview(self) -> None:
+ try:
+ settings = self._camera_settings_from_ui()
+ except ValueError as exc:
+ self._show_error(str(exc))
+ return
+ self.camera_controller.start(settings)
+ self.preview_button.setEnabled(False)
+ self.stop_preview_button.setEnabled(True)
+ self.statusBar().showMessage("Camera preview started", 3000)
+ if self.enable_dlc_checkbox.isChecked():
+ self._configure_dlc()
+ else:
+ self._last_pose = None
+
+ def _stop_preview(self) -> None:
+ self.camera_controller.stop()
+ self.preview_button.setEnabled(True)
+ self.stop_preview_button.setEnabled(False)
+ self._current_frame = None
+ self._last_pose = None
+ self.video_label.setPixmap(QPixmap())
+ self.video_label.setText("Camera preview not started")
+ self.statusBar().showMessage("Camera preview stopped", 3000)
+
+ def _on_camera_stopped(self) -> None:
+ self.preview_button.setEnabled(True)
+ self.stop_preview_button.setEnabled(False)
+
+ def _configure_dlc(self) -> None:
+ try:
+ settings = self._dlc_settings_from_ui()
+ except (ValueError, json.JSONDecodeError) as exc:
+ self._show_error(f"Invalid DLCLive settings: {exc}")
+ self.enable_dlc_checkbox.setChecked(False)
+ return
+ self.dlc_processor.configure(settings)
+
+ # ------------------------------------------------------------------ recording
+ def _start_recording(self) -> None:
+ if self._video_recorder and self._video_recorder.is_running:
+ return
+ try:
+ recording = self._recording_settings_from_ui()
+ except json.JSONDecodeError as exc:
+ self._show_error(f"Invalid recording options: {exc}")
+ return
+ if not recording.enabled:
+ self._show_error("Recording is disabled in the configuration.")
+ return
+ output_path = recording.output_path()
+ self._video_recorder = VideoRecorder(output_path, recording.options)
+ try:
+ self._video_recorder.start()
+ except Exception as exc: # pragma: no cover - runtime error
+ self._show_error(str(exc))
+ self._video_recorder = None
+ return
+ self.start_record_button.setEnabled(False)
+ self.stop_record_button.setEnabled(True)
+ self.statusBar().showMessage(f"Recording to {output_path}", 5000)
+
+ def _stop_recording(self) -> None:
+ if not self._video_recorder:
+ return
+ self._video_recorder.stop()
+ self._video_recorder = None
+ self.start_record_button.setEnabled(True)
+ self.stop_record_button.setEnabled(False)
+ self.statusBar().showMessage("Recording stopped", 3000)
+
+ # ------------------------------------------------------------------ frame handling
+ def _on_frame_ready(self, frame_data: FrameData) -> None:
+ frame = frame_data.image
+ self._current_frame = frame
+ if self._video_recorder and self._video_recorder.is_running:
+ self._video_recorder.write(frame)
+ if self.enable_dlc_checkbox.isChecked():
+ self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
+ self._update_video_display(frame)
+
+ def _on_pose_ready(self, result: PoseResult) -> None:
+ self._last_pose = result
+ if self._current_frame is not None:
+ self._update_video_display(self._current_frame)
+
+ def _update_video_display(self, frame: np.ndarray) -> None:
+ display_frame = frame
+ if self._last_pose and self._last_pose.pose is not None:
+ display_frame = self._draw_pose(frame, self._last_pose.pose)
+ rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB)
+ h, w, ch = rgb.shape
+ bytes_per_line = ch * w
+ image = QImage(rgb.data, w, h, bytes_per_line, QImage.Format.Format_RGB888)
+ self.video_label.setPixmap(QPixmap.fromImage(image))
+
+ def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
+ overlay = frame.copy()
+ for keypoint in np.asarray(pose):
+ if len(keypoint) < 2:
+ continue
+ x, y = keypoint[:2]
+ if np.isnan(x) or np.isnan(y):
+ continue
+ cv2.circle(overlay, (int(x), int(y)), 4, (0, 255, 0), -1)
+ return overlay
+
+ def _on_dlc_initialised(self, success: bool) -> None:
+ if success:
+ self.statusBar().showMessage("DLCLive initialised", 3000)
+ else:
+ self.statusBar().showMessage("DLCLive initialisation failed", 3000)
+
+ # ------------------------------------------------------------------ helpers
+ def _show_error(self, message: str) -> None:
+ self.statusBar().showMessage(message, 5000)
+ QMessageBox.critical(self, "Error", message)
+
+ # ------------------------------------------------------------------ Qt overrides
+ def closeEvent(self, event: QCloseEvent) -> None: # pragma: no cover - GUI behaviour
+ if self.camera_controller.is_running():
+ self.camera_controller.stop()
+ if self._video_recorder and self._video_recorder.is_running:
+ self._video_recorder.stop()
+ self.dlc_processor.shutdown()
+ super().closeEvent(event)
+
+
+def main() -> None:
+ app = QApplication(sys.argv)
+ window = MainWindow()
+ window.show()
+ sys.exit(app.exec())
+
+
+if __name__ == "__main__": # pragma: no cover - manual start
+ main()
diff --git a/dlclivegui/pose_process.py b/dlclivegui/pose_process.py
deleted file mode 100644
index 7ae4809..0000000
--- a/dlclivegui/pose_process.py
+++ /dev/null
@@ -1,273 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import multiprocess as mp
-import threading
-import time
-import pandas as pd
-import numpy as np
-
-from dlclivegui import CameraProcess
-from dlclivegui.queue import ClearableQueue, ClearableMPQueue
-
-
-class DLCLiveProcessError(Exception):
- """
- Exception for incorrect use of DLC-live-GUI Process Manager
- """
-
- pass
-
-
-class CameraPoseProcess(CameraProcess):
- """ Camera Process Manager class. Controls image capture, pose estimation and writing images to a video file in a background process.
-
- Parameters
- ----------
- device : :class:`cameracontrol.Camera`
- a camera object
- ctx : :class:`multiprocess.Context`
- multiprocessing context
- """
-
- def __init__(self, device, ctx=mp.get_context("spawn")):
- """ Constructor method
- """
-
- super().__init__(device, ctx)
- self.display_pose = None
- self.display_pose_queue = ClearableMPQueue(2, ctx=self.ctx)
- self.pose_process = None
-
- def start_pose_process(self, dlc_params, timeout=300):
-
- self.pose_process = self.ctx.Process(
- target=self._run_pose,
- args=(self.frame_shared, self.frame_time_shared, dlc_params),
- daemon=True,
- )
- self.pose_process.start()
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "pose") and (cmd[1] == "start"):
- return cmd[2]
- else:
- self.q_to_process.write(cmd)
-
- def _run_pose(self, frame_shared, frame_time, dlc_params):
-
- res = self.device.im_size
- self.frame = np.frombuffer(frame_shared.get_obj(), dtype="uint8").reshape(
- res[1], res[0], 3
- )
- self.frame_time = np.frombuffer(frame_time.get_obj(), dtype="d")
-
- ret = self._open_dlc_live(dlc_params)
- self.q_from_process.write(("pose", "start", ret))
-
- self._pose_loop()
- self.q_from_process.write(("pose", "end"))
-
- def _open_dlc_live(self, dlc_params):
-
- from dlclive import DLCLive
-
- ret = False
-
- self.opt_rate = True if dlc_params.pop("mode") == "Optimize Rate" else False
-
- proc_params = dlc_params.pop("processor")
- if proc_params is not None:
- proc_obj = proc_params.pop("object", None)
- if proc_obj is not None:
- dlc_params["processor"] = proc_obj(**proc_params)
-
- self.dlc = DLCLive(**dlc_params)
- if self.frame is not None:
- self.dlc.init_inference(
- self.frame, frame_time=self.frame_time[0], record=False
- )
- self.poses = []
- self.pose_times = []
- self.pose_frame_times = []
- ret = True
-
- return ret
-
- def _pose_loop(self):
- """ Conduct pose estimation using deeplabcut-live in loop
- """
-
- run = True
- write = False
- frame_time = 0
- pose_time = 0
- end_time = time.time()
-
- while run:
-
- ref_time = frame_time if self.opt_rate else end_time
-
- if self.frame_time[0] > ref_time:
-
- frame = self.frame
- frame_time = self.frame_time[0]
- pose = self.dlc.get_pose(frame, frame_time=frame_time, record=write)
- pose_time = time.time()
-
- self.display_pose_queue.write(pose, clear=True)
-
- if write:
- self.poses.append(pose)
- self.pose_times.append(pose_time)
- self.pose_frame_times.append(frame_time)
-
- cmd = self.q_to_process.read()
- if cmd is not None:
- if cmd[0] == "pose":
- if cmd[1] == "write":
- write = cmd[2]
- self.q_from_process.write(cmd)
- elif cmd[1] == "save":
- ret = self._save_pose(cmd[2])
- self.q_from_process.write(cmd + (ret,))
- elif cmd[1] == "end":
- run = False
- else:
- self.q_to_process.write(cmd)
-
- def start_record(self, timeout=5):
-
- ret = super().start_record(timeout=timeout)
-
- if (self.pose_process is not None) and (self.writer_process is not None):
- if (self.pose_process.is_alive()) and (self.writer_process.is_alive()):
- self.q_to_process.write(("pose", "write", True))
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "pose") and (cmd[1] == "write"):
- ret = cmd[2]
- break
- else:
- self.q_from_process.write(cmd)
-
- return ret
-
- def stop_record(self, timeout=5):
-
- ret = super().stop_record(timeout=timeout)
-
- if (self.pose_process is not None) and (self.writer_process is not None):
- if (self.pose_process.is_alive()) and (self.writer_process.is_alive()):
- self.q_to_process.write(("pose", "write", False))
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "pose") and (cmd[1] == "write"):
- ret = not cmd[2]
- break
- else:
- self.q_from_process.write(cmd)
-
- return ret
-
- def stop_pose_process(self):
-
- ret = True
- if self.pose_process is not None:
- if self.pose_process.is_alive():
- self.q_to_process.write(("pose", "end"))
-
- while True:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if cmd[0] == "pose":
- if cmd[1] == "end":
- break
- else:
- self.q_from_process.write(cmd)
-
- self.pose_process.join(5)
- if self.pose_process.is_alive():
- self.pose_process.terminate()
-
- return True
-
- def save_pose(self, filename, timeout=60):
-
- ret = False
- if self.pose_process is not None:
- if self.pose_process.is_alive():
- self.q_to_process.write(("pose", "save", filename))
-
- stime = time.time()
- while time.time() - stime < timeout:
- cmd = self.q_from_process.read()
- if cmd is not None:
- if (cmd[0] == "pose") and (cmd[1] == "save"):
- ret = cmd[3]
- break
- else:
- self.q_from_process.write(cmd)
- return ret
-
- def _save_pose(self, filename):
- """ Saves a pandas data frame with pose data collected while recording video
-
- Returns
- -------
- bool
- a logical flag indicating whether save was successful
- """
-
- ret = False
-
- if len(self.pose_times) > 0:
-
- dlc_file = f"{filename}_DLC.hdf5"
- proc_file = f"{filename}_PROC"
-
- bodyparts = self.dlc.cfg["all_joints_names"]
- poses = np.array(self.poses)
- poses = poses.reshape((poses.shape[0], poses.shape[1] * poses.shape[2]))
- pdindex = pd.MultiIndex.from_product(
- [bodyparts, ["x", "y", "likelihood"]], names=["bodyparts", "coords"]
- )
- pose_df = pd.DataFrame(poses, columns=pdindex)
- pose_df["frame_time"] = self.pose_frame_times
- pose_df["pose_time"] = self.pose_times
-
- pose_df.to_hdf(dlc_file, key="df_with_missing", mode="w")
- if self.dlc.processor is not None:
- self.dlc.processor.save(proc_file)
-
- self.poses = []
- self.pose_times = []
- self.pose_frame_times = []
-
- ret = True
-
- return ret
-
- def get_display_pose(self):
-
- pose = self.display_pose_queue.read(clear=True)
- if pose is not None:
- self.display_pose = pose
- if self.device.display_resize != 1:
- self.display_pose[:, :2] *= self.device.display_resize
-
- return self.display_pose
diff --git a/dlclivegui/processor/__init__.py b/dlclivegui/processor/__init__.py
deleted file mode 100644
index b97a9cc..0000000
--- a/dlclivegui/processor/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .teensy_laser.teensy_laser import TeensyLaser
diff --git a/dlclivegui/processor/processor.py b/dlclivegui/processor/processor.py
deleted file mode 100644
index 05eb7a8..0000000
--- a/dlclivegui/processor/processor.py
+++ /dev/null
@@ -1,23 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-"""
-Default processor class. Processors must contain two methods:
-i) process: takes in a pose, performs operations, and returns a pose
-ii) save: saves any internal data generated by the processor (such as timestamps for commands to external hardware)
-"""
-
-
-class Processor(object):
- def __init__(self):
- pass
-
- def process(self, pose):
- return pose
-
- def save(self, file=""):
- return 0
diff --git a/dlclivegui/processor/teensy_laser/__init__.py b/dlclivegui/processor/teensy_laser/__init__.py
deleted file mode 100644
index d2f10ca..0000000
--- a/dlclivegui/processor/teensy_laser/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .teensy_laser import *
diff --git a/dlclivegui/processor/teensy_laser/teensy_laser.ino b/dlclivegui/processor/teensy_laser/teensy_laser.ino
deleted file mode 100644
index 76a470b..0000000
--- a/dlclivegui/processor/teensy_laser/teensy_laser.ino
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Commands:
- * O = opto on; command = O, frequency, width, duration
- * X = opto off
- * R = reboot
- */
-
-
-const int opto_pin = 0;
-unsigned int opto_start = 0,
- opto_duty_cycle = 0,
- opto_freq = 0,
- opto_width = 0,
- opto_dur = 0;
-
-unsigned int read_int16() {
- union u_tag {
- byte b[2];
- unsigned int val;
- } par;
- for (int i=0; i<2; i++){
- if ((Serial.available() > 0))
- par.b[i] = Serial.read();
- else
- par.b[i] = 0;
- }
- return par.val;
-}
-
-void setup() {
- Serial.begin(115200);
- pinMode(opto_pin, OUTPUT);
-}
-
-void loop() {
-
- unsigned int curr_time = millis();
-
- while (Serial.available() > 0) {
-
- unsigned int cmd = Serial.read();
-
- if(cmd == 'O') {
-
- opto_start = curr_time;
- opto_freq = read_int16();
- opto_width = read_int16();
- opto_dur = read_int16();
- if (opto_dur == 0)
- opto_dur = 65355;
- opto_duty_cycle = opto_width * opto_freq * 4096 / 1000;
- analogWriteFrequency(opto_pin, opto_freq);
- analogWrite(opto_pin, opto_duty_cycle);
-
- Serial.print(opto_freq);
- Serial.print(',');
- Serial.print(opto_width);
- Serial.print(',');
- Serial.print(opto_dur);
- Serial.print('\n');
- Serial.flush();
-
- } else if(cmd == 'X') {
-
- analogWrite(opto_pin, 0);
-
- } else if(cmd == 'R') {
-
- _reboot_Teensyduino_();
-
- }
- }
-
- if (curr_time > opto_start + opto_dur)
- analogWrite(opto_pin, 0);
-
-}
diff --git a/dlclivegui/processor/teensy_laser/teensy_laser.py b/dlclivegui/processor/teensy_laser/teensy_laser.py
deleted file mode 100644
index 4535d55..0000000
--- a/dlclivegui/processor/teensy_laser/teensy_laser.py
+++ /dev/null
@@ -1,77 +0,0 @@
-from ..processor import Processor
-import serial
-import struct
-import time
-
-
-class TeensyLaser(Processor):
- def __init__(
- self, com, baudrate=115200, pulse_freq=50, pulse_width=5, max_stim_dur=0
- ):
-
- super().__init__()
- self.ser = serial.Serial(com, baudrate)
- self.pulse_freq = pulse_freq
- self.pulse_width = pulse_width
- self.max_stim_dur = (
- max_stim_dur if (max_stim_dur >= 0) and (max_stim_dur < 65356) else 0
- )
- self.stim_on = False
- self.stim_on_time = []
- self.stim_off_time = []
-
- def close_serial(self):
-
- self.ser.close()
-
- def stimulate_on(self):
-
- # command to activate PWM signal to laser is the letter 'O' followed by three 16 bit integers -- pulse frequency, pulse width, and max stim duration
- if not self.stim_on:
- self.ser.write(
- b"O"
- + struct.pack(
- "HHH", self.pulse_freq, self.pulse_width, self.max_stim_dur
- )
- )
- self.stim_on = True
- self.stim_on_time.append(time.time())
-
- def stim_off(self):
-
- # command to turn off PWM signal to laser is the letter 'X'
- if self.stim_on:
- self.ser.write(b"X")
- self.stim_on = False
- self.stim_off_time.append(time.time())
-
- def process(self, pose):
-
- # define criteria to stimulate (e.g. if first point is in a corner of the video)
- box = [[0, 100], [0, 100]]
- if (
- (pose[0][0] > box[0][0])
- and (pose[0][0] < box[0][1])
- and (pose[0][1] > box[1][0])
- and (pose[0][1] < box[1][1])
- ):
- self.stimulate_on()
- else:
- self.stim_off()
-
- return pose
-
- def save(self, file=None):
-
- ### save stim on and stim off times
- save_code = 0
- if file:
- try:
- pickle.dump(
- {"stim_on": self.stim_on_time, "stim_off": self.stim_off_time},
- open(file, "wb"),
- )
- save_code = 1
- except Exception:
- save_code = -1
- return save_code
diff --git a/dlclivegui/queue.py b/dlclivegui/queue.py
deleted file mode 100644
index 59bc43c..0000000
--- a/dlclivegui/queue.py
+++ /dev/null
@@ -1,208 +0,0 @@
-import multiprocess as mp
-from multiprocess import queues
-from queue import Queue, Empty, Full
-
-
-class QueuePositionError(Exception):
- """ Error in position argument of queue read """
-
- pass
-
-
-class ClearableQueue(Queue):
- """ A Queue that provides safe methods for writing to a full queue, reading to an empty queue, and a method to clear the queue """
-
- def __init__(self, maxsize=0):
-
- super().__init__(maxsize)
-
- def clear(self):
- """ Clears queue, returns all objects in a list
-
- Returns
- -------
- list
- list of objects from the queue
- """
-
- objs = []
-
- try:
- while True:
- objs.append(self.get_nowait())
- except Empty:
- pass
-
- return objs
-
- def write(self, obj, clear=False):
- """ Puts an object in the queue, with an option to clear queue before writing.
-
- Parameters
- ----------
- obj : [type]
- An object to put in the queue
- clear : bool, optional
- flag to clear queue before putting, by default False
-
- Returns
- -------
- bool
- if write was sucessful, returns True
- """
-
- if clear:
- self.clear()
-
- try:
- self.put_nowait(obj)
- success = True
- except Full:
- success = False
-
- return success
-
- def read(self, clear=False, position="last"):
- """ Gets an object in the queue, with the option to clear the queue and return the first element, last element, or all elements
-
- Parameters
- ----------
- clear : bool, optional
- flag to clear queue before putting, by default False
- position : str, optional
- If clear is True, returned object depends on position.
- If position = "last", returns last object.
- If position = "first", returns first object.
- If position = "all", returns all objects from the queue.
-
- Returns
- -------
- object
- object retrieved from the queue
- """
-
- obj = None
-
- if clear:
-
- objs = self.clear()
-
- if len(objs) > 0:
- if position == "first":
- obj = objs[0]
- elif position == "last":
- obj = objs[-1]
- elif position == "all":
- obj = objs
- else:
- raise QueuePositionError(
- "Queue read position should be one of 'first', 'last', or 'all'"
- )
- else:
-
- try:
- obj = self.get_nowait()
- except Empty:
- pass
-
- return obj
-
-
-class ClearableMPQueue(mp.queues.Queue):
- """ A multiprocess Queue that provides safe methods for writing to a full queue, reading to an empty queue, and a method to clear the queue """
-
- def __init__(self, maxsize=0, ctx=mp.get_context("spawn")):
-
- super().__init__(maxsize, ctx=ctx)
-
- def clear(self):
- """ Clears queue, returns all objects in a list
-
- Returns
- -------
- list
- list of objects from the queue
- """
-
- objs = []
-
- try:
- while True:
- objs.append(self.get_nowait())
- except Empty:
- pass
-
- return objs
-
- def write(self, obj, clear=False):
- """ Puts an object in the queue, with an option to clear queue before writing.
-
- Parameters
- ----------
- obj : [type]
- An object to put in the queue
- clear : bool, optional
- flag to clear queue before putting, by default False
-
- Returns
- -------
- bool
- if write was sucessful, returns True
- """
-
- if clear:
- self.clear()
-
- try:
- self.put_nowait(obj)
- success = True
- except Full:
- success = False
-
- return success
-
- def read(self, clear=False, position="last"):
- """ Gets an object in the queue, with the option to clear the queue and return the first element, last element, or all elements
-
- Parameters
- ----------
- clear : bool, optional
- flag to clear queue before putting, by default False
- position : str, optional
- If clear is True, returned object depends on position.
- If position = "last", returns last object.
- If position = "first", returns first object.
- If position = "all", returns all objects from the queue.
-
- Returns
- -------
- object
- object retrieved from the queue
- """
-
- obj = None
-
- if clear:
-
- objs = self.clear()
-
- if len(objs) > 0:
- if position == "first":
- obj = objs[0]
- elif position == "last":
- obj = objs[-1]
- elif position == "all":
- obj = objs
- else:
- raise QueuePositionError(
- "Queue read position should be one of 'first', 'last', or 'all'"
- )
-
- else:
-
- try:
- obj = self.get_nowait()
- except Empty:
- pass
-
- return obj
diff --git a/dlclivegui/tkutil.py b/dlclivegui/tkutil.py
deleted file mode 100644
index 3a5837e..0000000
--- a/dlclivegui/tkutil.py
+++ /dev/null
@@ -1,195 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import tkinter as tk
-from tkinter import ttk
-from distutils.util import strtobool
-
-
-class SettingsWindow(tk.Toplevel):
- def __init__(
- self,
- title="Edit Settings",
- settings={},
- names=None,
- vals=None,
- dtypes=None,
- restrictions=None,
- parent=None,
- ):
- """ Create a tkinter settings window
-
- Parameters
- ----------
- title : str, optional
- title for window
- settings : dict, optional
- dictionary of settings with keys = setting names.
- The value for each setting should be a dictionary with three keys:
- value (a default value),
- dtype (the data type for the setting),
- restriction (a list of possible values the parameter can take on)
- names : list, optional
- list of setting names, by default None
- vals : list, optional
- list of default values, by default None
- dtypes : list, optional
- list of setting data types, by default None
- restrictions : dict, optional
- dictionary of setting value restrictions, with keys = setting name and value = list of restrictions, by default {}
- parent : :class:`tkinter.Tk`, optional
- parent window, by default None
-
- Raises
- ------
- ValueError
- throws error if neither settings dictionary nor setting names are provided
- """
-
- super().__init__(parent)
- self.title(title)
-
- if settings:
- self.settings = settings
- elif not names:
- raise ValueError(
- "No argument names or settings dictionary. One must be provided to create a SettingsWindow."
- )
- else:
- self.settings = self.create_settings_dict(names, vals, dtypes, restrictions)
-
- self.cur_row = 0
- self.combobox_width = 15
-
- self.create_window()
-
- def create_settings_dict(self, names, vals=None, dtypes=None, restrictions=None):
- """Create dictionary of settings from names, vals, dtypes, and restrictions
-
- Parameters
- ----------
- names : list
- list of setting names
- vals : list
- list of default setting values
- dtypes : list
- list of setting dtype
- restrictions : dict
- dictionary of settting restrictions
-
- Returns
- -------
- dict
- settings dictionary with keys = names and value = dictionary with value, dtype, restrictions
- """
-
- set_dict = {}
- for i in range(len(names)):
-
- dt = dtypes[i] if dtypes is not None else None
-
- if vals is not None:
- val = dt(val) if type(dt) is type else [dt[0](v) for v in val]
- else:
- val = None
-
- restrict = restrictions[names[i]] if restrictions is not None else None
-
- set_dict[names[i]] = {"value": val, "dtype": dt, "restriction": restrict}
-
- return set_dict
-
- def create_window(self):
- """ Create settings GUI widgets
- """
-
- self.entry_vars = []
- names = tuple(self.settings.keys())
- for i in range(len(names)):
-
- this_setting = self.settings[names[i]]
-
- tk.Label(self, text=names[i] + ": ").grid(row=self.cur_row, column=0)
-
- v = this_setting["value"]
- if type(this_setting["dtype"]) is list:
- v = [str(x) if x is not None else "" for x in v]
- v = ", ".join(v)
- else:
- v = str(v) if v is not None else ""
- self.entry_vars.append(tk.StringVar(self, value=v))
-
- use_restriction = False
- if "restriction" in this_setting:
- if this_setting["restriction"] is not None:
- use_restriction = True
-
- if use_restriction:
- ttk.Combobox(
- self,
- textvariable=self.entry_vars[-1],
- values=this_setting["restriction"],
- state="readonly",
- width=self.combobox_width,
- ).grid(sticky="nsew", row=self.cur_row, column=1)
- else:
- tk.Entry(self, textvariable=self.entry_vars[-1]).grid(
- sticky="nsew", row=self.cur_row, column=1
- )
-
- self.cur_row += 1
-
- self.cur_row += 1
- tk.Button(self, text="Update", command=self.update_vals).grid(
- sticky="nsew", row=self.cur_row, column=1
- )
- self.cur_row += 1
- tk.Button(self, text="Cancel", command=self.destroy).grid(
- sticky="nsew", row=self.cur_row, column=1
- )
-
- _, row_count = self.grid_size()
- for r in range(row_count):
- self.grid_rowconfigure(r, minsize=20)
-
- def update_vals(self):
-
- names = tuple(self.settings.keys())
-
- for i in range(len(self.entry_vars)):
-
- name = names[i]
- val = self.entry_vars[i].get()
- dt = (
- self.settings[name]["dtype"] if "dtype" in self.settings[name] else None
- )
-
- val = [v.strip() for v in val.split(",")]
- use_dt = dt if type(dt) is type else dt[0]
- use_dt = strtobool if use_dt is bool else use_dt
-
- try:
- val = [use_dt(v) if v else None for v in val]
- except TypeError:
- pass
-
- val = val if type(dt) is list else val[0]
-
- self.settings[name]["value"] = val
-
- self.quit()
- self.destroy()
-
- def get_values(self):
-
- val_dict = {}
- names = tuple(self.settings.keys())
- for i in range(len(self.settings)):
- val_dict[names[i]] = self.settings[names[i]]["value"]
-
- return val_dict
diff --git a/dlclivegui/video.py b/dlclivegui/video.py
deleted file mode 100644
index d05f5b8..0000000
--- a/dlclivegui/video.py
+++ /dev/null
@@ -1,274 +0,0 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
-
-import os
-import numpy as np
-import pandas as pd
-import cv2
-import colorcet as cc
-from PIL import ImageColor
-from tqdm import tqdm
-
-
-def create_labeled_video(
- data_dir,
- out_dir=None,
- dlc_online=True,
- save_images=False,
- cut=(0, np.Inf),
- crop=None,
- cmap="bmy",
- radius=3,
- lik_thresh=0.5,
- write_ts=False,
- write_scale=2,
- write_pos="bottom-left",
- write_ts_offset=0,
- display=False,
- progress=True,
- label=True,
-):
- """ Create a labeled video from DeepLabCut-live-GUI recording
-
- Parameters
- ----------
- data_dir : str
- path to data directory
- dlc_online : bool, optional
- flag indicating dlc keypoints from online tracking, using DeepLabCut-live-GUI, or offline tracking, using :func:`dlclive.benchmark_videos`
- out_file : str, optional
- path for output file. If None, output file will be "'video_file'_LABELED.avi". by default None. If NOn
- save_images : bool, optional
- boolean flag to save still images in a folder
- cut : tuple, optional
- time of video to use. Will only save labeled video for time after cut[0] and before cut[1], by default (0, np.Inf)
- cmap : str, optional
- a :package:`colorcet` colormap, by default 'bmy'
- radius : int, optional
- radius for keypoints, by default 3
- lik_thresh : float, optional
- likelihood threshold to plot keypoints, by default 0.5
- display : bool, optional
- boolean flag to display images as video is written, by default False
- progress : bool, optional
- boolean flag to display progress bar
-
- Raises
- ------
- Exception
- if frames cannot be read from the video file
- """
-
- base_dir = os.path.basename(data_dir)
- video_file = os.path.normpath(f"{data_dir}/{base_dir}_VIDEO.avi")
- ts_file = os.path.normpath(f"{data_dir}/{base_dir}_TS.npy")
- dlc_file = (
- os.path.normpath(f"{data_dir}/{base_dir}_DLC.hdf5")
- if dlc_online
- else os.path.normpath(f"{data_dir}/{base_dir}_VIDEO_DLCLIVE_POSES.h5")
- )
-
- cap = cv2.VideoCapture(video_file)
- cam_frame_times = np.load(ts_file)
- n_frames = cam_frame_times.size
-
- lab = "LABELED" if label else "UNLABELED"
- if out_dir:
- out_file = (
- f"{out_dir}/{os.path.splitext(os.path.basename(video_file))[0]}_{lab}.avi"
- )
- out_times_file = (
- f"{out_dir}/{os.path.splitext(os.path.basename(ts_file))[0]}_{lab}.npy"
- )
- else:
- out_file = f"{os.path.splitext(video_file)[0]}_{lab}.avi"
- out_times_file = f"{os.path.splitext(ts_file)[0]}_{lab}.npy"
-
- os.makedirs(os.path.normpath(os.path.dirname(out_file)), exist_ok=True)
-
- if save_images:
- im_dir = os.path.splitext(out_file)[0]
- os.makedirs(im_dir, exist_ok=True)
-
- im_size = (
- int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
- int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
- )
- if crop is not None:
- crop[0] = crop[0] if crop[0] > 0 else 0
- crop[1] = crop[1] if crop[1] > 0 else im_size[1]
- crop[2] = crop[2] if crop[2] > 0 else 0
- crop[3] = crop[3] if crop[3] > 0 else im_size[0]
- im_size = (crop[3] - crop[2], crop[1] - crop[0])
-
- fourcc = cv2.VideoWriter_fourcc(*"DIVX")
- fps = cap.get(cv2.CAP_PROP_FPS)
- vwriter = cv2.VideoWriter(out_file, fourcc, fps, im_size)
- label_times = []
-
- if write_ts:
- ts_font = cv2.FONT_HERSHEY_PLAIN
-
- if "left" in write_pos:
- ts_w = 0
- else:
- ts_w = (
- im_size[0] if crop is None else (crop[3] - crop[2]) - (55 * write_scale)
- )
-
- if "bottom" in write_pos:
- ts_h = im_size[1] if crop is None else (crop[1] - crop[0])
- else:
- ts_h = 0 if crop is None else crop[0] + (12 * write_scale)
-
- ts_coord = (ts_w, ts_h)
- ts_color = (255, 255, 255)
- ts_size = 2
-
- poses = pd.read_hdf(dlc_file)
- if dlc_online:
- pose_times = poses["pose_time"]
- else:
- poses["frame_time"] = cam_frame_times
- poses["pose_time"] = cam_frame_times
- poses = poses.melt(id_vars=["frame_time", "pose_time"])
- bodyparts = poses["bodyparts"].unique()
-
- all_colors = getattr(cc, cmap)
- colors = [
- ImageColor.getcolor(c, "RGB")[::-1]
- for c in all_colors[:: int(len(all_colors) / bodyparts.size)]
- ]
-
- ind = 0
- vid_time = 0
- while vid_time < cut[0]:
-
- cur_time = cam_frame_times[ind]
- vid_time = cur_time - cam_frame_times[0]
- ret, frame = cap.read()
- ind += 1
-
- if not ret:
- raise Exception(
- f"Could not read frame = {ind+1} at time = {cur_time-cam_frame_times[0]}."
- )
-
- frame_times_sub = cam_frame_times[
- (cam_frame_times - cam_frame_times[0] > cut[0])
- & (cam_frame_times - cam_frame_times[0] < cut[1])
- ]
- iterator = (
- tqdm(range(ind, ind + frame_times_sub.size))
- if progress
- else range(ind, ind + frame_times_sub.size)
- )
- this_pose = np.zeros((bodyparts.size, 3))
-
- for i in iterator:
-
- cur_time = cam_frame_times[i]
- vid_time = cur_time - cam_frame_times[0]
- ret, frame = cap.read()
-
- if not ret:
- raise Exception(
- f"Could not read frame = {i+1} at time = {cur_time-cam_frame_times[0]}."
- )
-
- if dlc_online:
- poses_before_index = np.where(pose_times < cur_time)[0]
- if poses_before_index.size > 0:
- cur_pose_time = pose_times[poses_before_index[-1]]
- this_pose = poses[poses["pose_time"] == cur_pose_time]
- else:
- this_pose = poses[poses["frame_time"] == cur_time]
-
- if label:
- for j in range(bodyparts.size):
- this_bp = this_pose[this_pose["bodyparts"] == bodyparts[j]][
- "value"
- ].values
- if this_bp[2] > lik_thresh:
- x = int(this_bp[0])
- y = int(this_bp[1])
- frame = cv2.circle(frame, (x, y), radius, colors[j], thickness=-1)
-
- if crop is not None:
- frame = frame[crop[0] : crop[1], crop[2] : crop[3]]
-
- if write_ts:
- frame = cv2.putText(
- frame,
- f"{(vid_time-write_ts_offset):0.3f}",
- ts_coord,
- ts_font,
- write_scale,
- ts_color,
- ts_size,
- )
-
- if display:
- cv2.imshow("DLC Live Labeled Video", frame)
- cv2.waitKey(1)
-
- vwriter.write(frame)
- label_times.append(cur_time)
- if save_images:
- new_file = f"{im_dir}/frame_{i}.png"
- cv2.imwrite(new_file, frame)
-
- if display:
- cv2.destroyAllWindows()
-
- vwriter.release()
- np.save(out_times_file, label_times)
-
-
-def main():
-
- import argparse
- import os
-
- parser = argparse.ArgumentParser()
- parser.add_argument("dir", type=str)
- parser.add_argument("-o", "--out-dir", type=str, default=None)
- parser.add_argument("--dlc-offline", action="store_true")
- parser.add_argument("-s", "--save-images", action="store_true")
- parser.add_argument("-u", "--cut", nargs="+", type=float, default=[0, np.Inf])
- parser.add_argument("-c", "--crop", nargs="+", type=int, default=None)
- parser.add_argument("-m", "--cmap", type=str, default="bmy")
- parser.add_argument("-r", "--radius", type=int, default=3)
- parser.add_argument("-l", "--lik-thresh", type=float, default=0.5)
- parser.add_argument("-w", "--write-ts", action="store_true")
- parser.add_argument("--write-scale", type=int, default=2)
- parser.add_argument("--write-pos", type=str, default="bottom-left")
- parser.add_argument("--write-ts-offset", type=float, default=0.0)
- parser.add_argument("-d", "--display", action="store_true")
- parser.add_argument("--no-progress", action="store_false")
- parser.add_argument("--no-label", action="store_false")
- args = parser.parse_args()
-
- create_labeled_video(
- args.dir,
- out_dir=args.out_dir,
- dlc_online=(not args.dlc_offline),
- save_images=args.save_images,
- cut=tuple(args.cut),
- crop=args.crop,
- cmap=args.cmap,
- radius=args.radius,
- lik_thresh=args.lik_thresh,
- write_ts=args.write_ts,
- write_scale=args.write_scale,
- write_pos=args.write_pos,
- write_ts_offset=args.write_ts_offset,
- display=args.display,
- progress=args.no_progress,
- label=args.no_label,
- )
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
new file mode 100644
index 0000000..e0e3706
--- /dev/null
+++ b/dlclivegui/video_recorder.py
@@ -0,0 +1,46 @@
+"""Video recording support using the vidgear library."""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import numpy as np
+
+try:
+ from vidgear.gears import WriteGear
+except ImportError: # pragma: no cover - handled at runtime
+ WriteGear = None # type: ignore[assignment]
+
+
+class VideoRecorder:
+ """Thin wrapper around :class:`vidgear.gears.WriteGear`."""
+
+ def __init__(self, output: Path | str, options: Optional[Dict[str, Any]] = None):
+ self._output = Path(output)
+ self._options = options or {}
+ self._writer: Optional[WriteGear] = None
+
+ @property
+ def is_running(self) -> bool:
+ return self._writer is not None
+
+ def start(self) -> None:
+ if WriteGear is None:
+ raise RuntimeError(
+ "vidgear is required for video recording. Install it with 'pip install vidgear'."
+ )
+ if self._writer is not None:
+ return
+ self._output.parent.mkdir(parents=True, exist_ok=True)
+ self._writer = WriteGear(output_filename=str(self._output), logging=False, **self._options)
+
+ def write(self, frame: np.ndarray) -> None:
+ if self._writer is None:
+ return
+ self._writer.write(frame)
+
+ def stop(self) -> None:
+ if self._writer is None:
+ return
+ self._writer.close()
+ self._writer = None
diff --git a/setup.py b/setup.py
index 28eb6cc..163f8f0 100644
--- a/setup.py
+++ b/setup.py
@@ -1,36 +1,32 @@
-"""
-DeepLabCut Toolbox (deeplabcut.org)
-© A. & M. Mathis Labs
-
-Licensed under GNU Lesser General Public License v3.0
-"""
-
+"""Setup configuration for the DeepLabCut Live GUI."""
+from __future__ import annotations
import setuptools
-with open("README.md", "r") as fh:
+with open("README.md", "r", encoding="utf-8") as fh:
long_description = fh.read()
setuptools.setup(
name="deeplabcut-live-gui",
- version="1.0",
+ version="2.0",
author="A. & M. Mathis Labs",
author_email="adim@deeplabcut.org",
- description="GUI to run real time deeplabcut experiments",
+ description="PyQt-based GUI to run real time DeepLabCut experiments",
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/DeepLabCut/DeepLabCut-live-GUI",
- python_requires=">=3.5, <3.11",
+ python_requires=">=3.11",
install_requires=[
"deeplabcut-live",
- "pyserial",
- "pandas",
- "tables",
- "multiprocess",
- "imutils",
- "pillow",
- "tqdm",
+ "PyQt6",
+ "numpy",
+ "opencv-python",
+ "vidgear[core]",
],
+ extras_require={
+ "basler": ["pypylon"],
+ "gentl": ["pygobject"],
+ },
packages=setuptools.find_packages(),
include_package_data=True,
classifiers=(
@@ -40,8 +36,7 @@
),
entry_points={
"console_scripts": [
- "dlclivegui=dlclivegui.dlclivegui:main",
- "dlclivegui-video=dlclivegui.video:main",
+ "dlclivegui=dlclivegui.gui:main",
]
},
)
From 893b28ad3e01bf1b29613886b9527b6777e9f22a Mon Sep 17 00:00:00 2001
From: Artur <35294812+arturoptophys@users.noreply.github.com>
Date: Tue, 21 Oct 2025 14:55:15 +0200
Subject: [PATCH 02/26] Rearrange UI and improve camera controls
---
dlclivegui/camera_controller.py | 17 +-
dlclivegui/cameras/base.py | 5 +
dlclivegui/cameras/factory.py | 62 ++++++-
dlclivegui/cameras/opencv_backend.py | 16 ++
dlclivegui/dlc_processor.py | 20 +++
dlclivegui/gui.py | 255 +++++++++++++++++++++++----
dlclivegui/video_recorder.py | 2 +-
7 files changed, 330 insertions(+), 47 deletions(-)
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 3398566..c8a5aaf 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -1,12 +1,12 @@
"""Camera management for the DLC Live GUI."""
from __future__ import annotations
-import time
from dataclasses import dataclass
+from threading import Event
from typing import Optional
import numpy as np
-from PyQt6.QtCore import QMetaObject, QObject, QThread, Qt, pyqtSignal, pyqtSlot
+from PyQt6.QtCore import QObject, QThread, pyqtSignal, pyqtSlot
from .cameras import CameraFactory
from .cameras.base import CameraBackend
@@ -31,12 +31,12 @@ class CameraWorker(QObject):
def __init__(self, settings: CameraSettings):
super().__init__()
self._settings = settings
- self._running = False
+ self._stop_event = Event()
self._backend: Optional[CameraBackend] = None
@pyqtSlot()
def run(self) -> None:
- self._running = True
+ self._stop_event.clear()
try:
self._backend = CameraFactory.create(self._settings)
self._backend.open()
@@ -45,7 +45,7 @@ def run(self) -> None:
self.finished.emit()
return
- while self._running:
+ while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
except Exception as exc: # pragma: no cover - device specific
@@ -63,7 +63,7 @@ def run(self) -> None:
@pyqtSlot()
def stop(self) -> None:
- self._running = False
+ self._stop_event.set()
if self._backend is not None:
try:
self._backend.stop()
@@ -106,11 +106,8 @@ def stop(self) -> None:
if not self.is_running():
return
assert self._worker is not None
- QMetaObject.invokeMethod(
- self._worker, "stop", Qt.ConnectionType.QueuedConnection
- )
+ self._worker.stop()
assert self._thread is not None
- self._thread.quit()
self._thread.wait()
@pyqtSlot()
diff --git a/dlclivegui/cameras/base.py b/dlclivegui/cameras/base.py
index 6ae79dc..910331c 100644
--- a/dlclivegui/cameras/base.py
+++ b/dlclivegui/cameras/base.py
@@ -33,6 +33,11 @@ def stop(self) -> None:
# Most backends do not require additional handling, but subclasses may
# override when they need to interrupt blocking reads.
+ def device_name(self) -> str:
+ """Return a human readable name for the device currently in use."""
+
+ return self.settings.name
+
@abstractmethod
def open(self) -> None:
"""Open the capture device."""
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index e9704ef..c67f7bd 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -2,12 +2,21 @@
from __future__ import annotations
import importlib
-from typing import Dict, Iterable, Tuple, Type
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Tuple, Type
from ..config import CameraSettings
from .base import CameraBackend
+@dataclass
+class DetectedCamera:
+ """Information about a camera discovered during probing."""
+
+ index: int
+ label: str
+
+
_BACKENDS: Dict[str, Tuple[str, str]] = {
"opencv": ("dlclivegui.cameras.opencv_backend", "OpenCVCameraBackend"),
"basler": ("dlclivegui.cameras.basler_backend", "BaslerCameraBackend"),
@@ -38,6 +47,57 @@ def available_backends() -> Dict[str, bool]:
availability[name] = backend_cls.is_available()
return availability
+ @staticmethod
+ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
+ """Probe ``backend`` for available cameras.
+
+ Parameters
+ ----------
+ backend:
+ The backend identifier, e.g. ``"opencv"``.
+ max_devices:
+ Upper bound for the indices that should be probed.
+
+ Returns
+ -------
+ list of :class:`DetectedCamera`
+ Sorted list of detected cameras with human readable labels.
+ """
+
+ try:
+ backend_cls = CameraFactory._resolve_backend(backend)
+ except RuntimeError:
+ return []
+ if not backend_cls.is_available():
+ return []
+
+ detected: List[DetectedCamera] = []
+ for index in range(max_devices):
+ settings = CameraSettings(
+ name=f"Probe {index}",
+ index=index,
+ width=640,
+ height=480,
+ fps=30.0,
+ backend=backend,
+ properties={},
+ )
+ backend_instance = backend_cls(settings)
+ try:
+ backend_instance.open()
+ except Exception:
+ continue
+ else:
+ label = backend_instance.device_name()
+ detected.append(DetectedCamera(index=index, label=label))
+ finally:
+ try:
+ backend_instance.close()
+ except Exception:
+ pass
+ detected.sort(key=lambda camera: camera.index)
+ return detected
+
@staticmethod
def create(settings: CameraSettings) -> CameraBackend:
"""Instantiate a backend for ``settings``."""
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index a64e3f1..8497bfa 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -39,6 +39,22 @@ def close(self) -> None:
self._capture.release()
self._capture = None
+ def stop(self) -> None:
+ if self._capture is not None:
+ self._capture.release()
+ self._capture = None
+
+ def device_name(self) -> str:
+ base_name = "OpenCV"
+ if self._capture is not None and hasattr(self._capture, "getBackendName"):
+ try:
+ backend_name = self._capture.getBackendName()
+ except Exception: # pragma: no cover - backend specific
+ backend_name = ""
+ if backend_name:
+ base_name = backend_name
+ return f"{base_name} camera #{self.settings.index}"
+
def _configure_capture(self) -> None:
if self._capture is None:
return
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 5c199b8..0e1ef3e 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -45,6 +45,18 @@ def __init__(self) -> None:
def configure(self, settings: DLCProcessorSettings) -> None:
self._settings = settings
+ def reset(self) -> None:
+ """Cancel pending work and drop the current DLCLive instance."""
+
+ with self._lock:
+ if self._pending is not None and not self._pending.done():
+ self._pending.cancel()
+ self._pending = None
+ if self._init_future is not None and not self._init_future.done():
+ self._init_future.cancel()
+ self._init_future = None
+ self._dlc = None
+
def shutdown(self) -> None:
with self._lock:
if self._pending is not None:
@@ -106,6 +118,10 @@ def _on_initialised(self, future: Future[Any]) -> None:
except Exception as exc: # pragma: no cover - runtime behaviour
LOGGER.exception("Failed to initialise DLCLive", exc_info=exc)
self.error.emit(str(exc))
+ finally:
+ with self._lock:
+ if self._init_future is future:
+ self._init_future = None
def _run_inference(self, frame: np.ndarray, timestamp: float) -> PoseResult:
if self._dlc is None:
@@ -120,4 +136,8 @@ def _on_pose_ready(self, future: Future[Any]) -> None:
LOGGER.exception("Pose inference failed", exc_info=exc)
self.error.emit(str(exc))
return
+ finally:
+ with self._lock:
+ if self._pending is future:
+ self._pending = None
self.pose_ready.emit(result)
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 4eb3971..7f6bbff 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -24,6 +24,7 @@
QMessageBox,
QPlainTextEdit,
QPushButton,
+ QSizePolicy,
QSpinBox,
QDoubleSpinBox,
QStatusBar,
@@ -33,6 +34,7 @@
from .camera_controller import CameraController, FrameData
from .cameras import CameraFactory
+from .cameras.factory import DetectedCamera
from .config import (
ApplicationSettings,
CameraSettings,
@@ -53,8 +55,12 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._config = config or DEFAULT_CONFIG
self._config_path: Optional[Path] = None
self._current_frame: Optional[np.ndarray] = None
+ self._raw_frame: Optional[np.ndarray] = None
self._last_pose: Optional[PoseResult] = None
+ self._dlc_active: bool = False
self._video_recorder: Optional[VideoRecorder] = None
+ self._rotation_degrees: int = 0
+ self._detected_cameras: list[DetectedCamera] = []
self.camera_controller = CameraController()
self.dlc_processor = DLCLiveProcessor()
@@ -62,20 +68,25 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._setup_ui()
self._connect_signals()
self._apply_config(self._config)
+ self._update_inference_buttons()
# ------------------------------------------------------------------ UI
def _setup_ui(self) -> None:
central = QWidget()
- layout = QVBoxLayout(central)
+ layout = QHBoxLayout(central)
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
- layout.addWidget(self.video_label)
+ self.video_label.setSizePolicy(
+ QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding
+ )
- layout.addWidget(self._build_camera_group())
- layout.addWidget(self._build_dlc_group())
- layout.addWidget(self._build_recording_group())
+ controls_widget = QWidget()
+ controls_layout = QVBoxLayout(controls_widget)
+ controls_layout.addWidget(self._build_camera_group())
+ controls_layout.addWidget(self._build_dlc_group())
+ controls_layout.addWidget(self._build_recording_group())
button_bar = QHBoxLayout()
self.preview_button = QPushButton("Start Preview")
@@ -83,7 +94,15 @@ def _setup_ui(self) -> None:
self.stop_preview_button.setEnabled(False)
button_bar.addWidget(self.preview_button)
button_bar.addWidget(self.stop_preview_button)
- layout.addLayout(button_bar)
+ controls_layout.addLayout(button_bar)
+ controls_layout.addStretch(1)
+
+ preview_layout = QVBoxLayout()
+ preview_layout.addWidget(self.video_label)
+ preview_layout.addStretch(1)
+
+ layout.addWidget(controls_widget)
+ layout.addLayout(preview_layout, stretch=1)
self.setCentralWidget(central)
self.setStatusBar(QStatusBar())
@@ -113,10 +132,13 @@ def _build_camera_group(self) -> QGroupBox:
group = QGroupBox("Camera settings")
form = QFormLayout(group)
+ index_layout = QHBoxLayout()
self.camera_index = QComboBox()
self.camera_index.setEditable(True)
- self.camera_index.addItems([str(i) for i in range(5)])
- form.addRow("Camera index", self.camera_index)
+ index_layout.addWidget(self.camera_index)
+ self.refresh_cameras_button = QPushButton("Refresh")
+ index_layout.addWidget(self.refresh_cameras_button)
+ form.addRow("Camera", index_layout)
self.camera_width = QSpinBox()
self.camera_width.setRange(1, 7680)
@@ -148,6 +170,13 @@ def _build_camera_group(self) -> QGroupBox:
self.camera_properties_edit.setFixedHeight(60)
form.addRow("Advanced properties", self.camera_properties_edit)
+ self.rotation_combo = QComboBox()
+ self.rotation_combo.addItem("0° (default)", 0)
+ self.rotation_combo.addItem("90°", 90)
+ self.rotation_combo.addItem("180°", 180)
+ self.rotation_combo.addItem("270°", 270)
+ form.addRow("Rotation", self.rotation_combo)
+
return group
def _build_dlc_group(self) -> QGroupBox:
@@ -185,9 +214,18 @@ def _build_dlc_group(self) -> QGroupBox:
self.additional_options_edit.setFixedHeight(60)
form.addRow("Additional options", self.additional_options_edit)
- self.enable_dlc_checkbox = QCheckBox("Enable pose estimation")
- self.enable_dlc_checkbox.setChecked(True)
- form.addRow(self.enable_dlc_checkbox)
+ inference_buttons = QHBoxLayout()
+ self.start_inference_button = QPushButton("Start pose inference")
+ self.start_inference_button.setEnabled(False)
+ inference_buttons.addWidget(self.start_inference_button)
+ self.stop_inference_button = QPushButton("Stop pose inference")
+ self.stop_inference_button.setEnabled(False)
+ inference_buttons.addWidget(self.stop_inference_button)
+ form.addRow(inference_buttons)
+
+ self.show_predictions_checkbox = QCheckBox("Display pose predictions")
+ self.show_predictions_checkbox.setChecked(True)
+ form.addRow(self.show_predictions_checkbox)
return group
@@ -236,19 +274,29 @@ def _connect_signals(self) -> None:
self.stop_preview_button.clicked.connect(self._stop_preview)
self.start_record_button.clicked.connect(self._start_recording)
self.stop_record_button.clicked.connect(self._stop_recording)
+ self.refresh_cameras_button.clicked.connect(
+ lambda: self._refresh_camera_indices(keep_current=True)
+ )
+ self.camera_backend.currentIndexChanged.connect(self._on_backend_changed)
+ self.camera_backend.editTextChanged.connect(self._on_backend_changed)
+ self.rotation_combo.currentIndexChanged.connect(self._on_rotation_changed)
+ self.start_inference_button.clicked.connect(self._start_inference)
+ self.stop_inference_button.clicked.connect(lambda: self._stop_inference())
+ self.show_predictions_checkbox.stateChanged.connect(
+ self._on_show_predictions_changed
+ )
self.camera_controller.frame_ready.connect(self._on_frame_ready)
self.camera_controller.error.connect(self._show_error)
self.camera_controller.stopped.connect(self._on_camera_stopped)
self.dlc_processor.pose_ready.connect(self._on_pose_ready)
- self.dlc_processor.error.connect(self._show_error)
+ self.dlc_processor.error.connect(self._on_dlc_error)
self.dlc_processor.initialized.connect(self._on_dlc_initialised)
# ------------------------------------------------------------------ config
def _apply_config(self, config: ApplicationSettings) -> None:
camera = config.camera
- self.camera_index.setCurrentText(str(camera.index))
self.camera_width.setValue(int(camera.width))
self.camera_height.setValue(int(camera.height))
self.camera_fps.setValue(float(camera.fps))
@@ -258,6 +306,10 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.camera_backend.setCurrentIndex(index)
else:
self.camera_backend.setEditText(backend_name)
+ self._refresh_camera_indices(keep_current=False)
+ self._select_camera_by_index(
+ camera.index, fallback_text=camera.name or str(camera.index)
+ )
self.camera_properties_edit.setPlainText(
json.dumps(camera.properties, indent=2) if camera.properties else ""
)
@@ -289,20 +341,14 @@ def _current_config(self) -> ApplicationSettings:
)
def _camera_settings_from_ui(self) -> CameraSettings:
- index_text = self.camera_index.currentText().strip() or "0"
- try:
- index = int(index_text)
- except ValueError:
- raise ValueError("Camera index must be an integer") from None
- backend_data = self.camera_backend.currentData()
- backend_text = (
- backend_data
- if isinstance(backend_data, str) and backend_data
- else self.camera_backend.currentText().strip()
- )
+ index = self._current_camera_index_value()
+ if index is None:
+ raise ValueError("Camera selection must provide a numeric index")
+ backend_text = self._current_backend_name()
properties = self._parse_json(self.camera_properties_edit.toPlainText())
+ name_text = self.camera_index.currentText().strip()
return CameraSettings(
- name=f"Camera {index}",
+ name=name_text or f"Camera {index}",
index=index,
width=self.camera_width.value(),
height=self.camera_height.value(),
@@ -311,6 +357,65 @@ def _camera_settings_from_ui(self) -> CameraSettings:
properties=properties,
)
+ def _current_backend_name(self) -> str:
+ backend_data = self.camera_backend.currentData()
+ if isinstance(backend_data, str) and backend_data:
+ return backend_data
+ text = self.camera_backend.currentText().strip()
+ return text or "opencv"
+
+ def _refresh_camera_indices(
+ self, *_args: object, keep_current: bool = True
+ ) -> None:
+ backend = self._current_backend_name()
+ detected = CameraFactory.detect_cameras(backend)
+ debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
+ print(
+ f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
+ )
+ self._detected_cameras = detected
+ previous_index = self._current_camera_index_value()
+ previous_text = self.camera_index.currentText()
+ self.camera_index.blockSignals(True)
+ self.camera_index.clear()
+ for camera in detected:
+ self.camera_index.addItem(camera.label, camera.index)
+ if keep_current and previous_index is not None:
+ self._select_camera_by_index(previous_index, fallback_text=previous_text)
+ elif detected:
+ self.camera_index.setCurrentIndex(0)
+ else:
+ if keep_current and previous_text:
+ self.camera_index.setEditText(previous_text)
+ else:
+ self.camera_index.setEditText("")
+ self.camera_index.blockSignals(False)
+
+ def _select_camera_by_index(
+ self, index: int, fallback_text: Optional[str] = None
+ ) -> None:
+ self.camera_index.blockSignals(True)
+ for row in range(self.camera_index.count()):
+ if self.camera_index.itemData(row) == index:
+ self.camera_index.setCurrentIndex(row)
+ break
+ else:
+ text = fallback_text if fallback_text is not None else str(index)
+ self.camera_index.setEditText(text)
+ self.camera_index.blockSignals(False)
+
+ def _current_camera_index_value(self) -> Optional[int]:
+ data = self.camera_index.currentData()
+ if isinstance(data, int):
+ return data
+ text = self.camera_index.currentText().strip()
+ if not text:
+ return None
+ try:
+ return int(text)
+ except ValueError:
+ return None
+
def _parse_optional_int(self, value: str) -> Optional[int]:
text = value.strip()
if not text:
@@ -402,6 +507,18 @@ def _action_browse_directory(self) -> None:
if directory:
self.output_directory_edit.setText(directory)
+ def _on_backend_changed(self, *_args: object) -> None:
+ self._refresh_camera_indices(keep_current=False)
+
+ def _on_rotation_changed(self, _index: int) -> None:
+ data = self.rotation_combo.currentData()
+ self._rotation_degrees = int(data) if isinstance(data, int) else 0
+ if self._raw_frame is not None:
+ rotated = self._apply_rotation(self._raw_frame)
+ self._current_frame = rotated
+ self._last_pose = None
+ self._update_video_display(rotated)
+
# ------------------------------------------------------------------ camera control
def _start_preview(self) -> None:
try:
@@ -412,34 +529,77 @@ def _start_preview(self) -> None:
self.camera_controller.start(settings)
self.preview_button.setEnabled(False)
self.stop_preview_button.setEnabled(True)
+ self._current_frame = None
+ self._raw_frame = None
+ self._last_pose = None
+ self._dlc_active = False
self.statusBar().showMessage("Camera preview started", 3000)
- if self.enable_dlc_checkbox.isChecked():
- self._configure_dlc()
- else:
- self._last_pose = None
+ self._update_inference_buttons()
def _stop_preview(self) -> None:
self.camera_controller.stop()
+ self._stop_inference(show_message=False)
self.preview_button.setEnabled(True)
self.stop_preview_button.setEnabled(False)
self._current_frame = None
+ self._raw_frame = None
self._last_pose = None
self.video_label.setPixmap(QPixmap())
self.video_label.setText("Camera preview not started")
self.statusBar().showMessage("Camera preview stopped", 3000)
+ self._update_inference_buttons()
def _on_camera_stopped(self) -> None:
self.preview_button.setEnabled(True)
self.stop_preview_button.setEnabled(False)
+ self._stop_inference(show_message=False)
+ self._update_inference_buttons()
- def _configure_dlc(self) -> None:
+ def _configure_dlc(self) -> bool:
try:
settings = self._dlc_settings_from_ui()
except (ValueError, json.JSONDecodeError) as exc:
self._show_error(f"Invalid DLCLive settings: {exc}")
- self.enable_dlc_checkbox.setChecked(False)
- return
+ return False
+ if not settings.model_path:
+ self._show_error("Please select a DLCLive model before starting inference.")
+ return False
self.dlc_processor.configure(settings)
+ return True
+
+ def _update_inference_buttons(self) -> None:
+ preview_running = self.camera_controller.is_running()
+ self.start_inference_button.setEnabled(preview_running and not self._dlc_active)
+ self.stop_inference_button.setEnabled(preview_running and self._dlc_active)
+
+ def _start_inference(self) -> None:
+ if self._dlc_active:
+ self.statusBar().showMessage("Pose inference already running", 3000)
+ return
+ if not self.camera_controller.is_running():
+ self._show_error(
+ "Start the camera preview before running pose inference."
+ )
+ return
+ if not self._configure_dlc():
+ self._update_inference_buttons()
+ return
+ self.dlc_processor.reset()
+ self._last_pose = None
+ self._dlc_active = True
+ self.statusBar().showMessage("Starting pose inference…", 3000)
+ self._update_inference_buttons()
+
+ def _stop_inference(self, show_message: bool = True) -> None:
+ was_active = self._dlc_active
+ self._dlc_active = False
+ self.dlc_processor.reset()
+ self._last_pose = None
+ if self._current_frame is not None:
+ self._update_video_display(self._current_frame)
+ if was_active and show_message:
+ self.statusBar().showMessage("Pose inference stopped", 3000)
+ self._update_inference_buttons()
# ------------------------------------------------------------------ recording
def _start_recording(self) -> None:
@@ -476,22 +636,34 @@ def _stop_recording(self) -> None:
# ------------------------------------------------------------------ frame handling
def _on_frame_ready(self, frame_data: FrameData) -> None:
- frame = frame_data.image
+ raw_frame = frame_data.image
+ self._raw_frame = raw_frame
+ frame = self._apply_rotation(raw_frame)
self._current_frame = frame
if self._video_recorder and self._video_recorder.is_running:
self._video_recorder.write(frame)
- if self.enable_dlc_checkbox.isChecked():
+ if self._dlc_active:
self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
self._update_video_display(frame)
def _on_pose_ready(self, result: PoseResult) -> None:
+ if not self._dlc_active:
+ return
self._last_pose = result
if self._current_frame is not None:
self._update_video_display(self._current_frame)
+ def _on_dlc_error(self, message: str) -> None:
+ self._stop_inference(show_message=False)
+ self._show_error(message)
+
def _update_video_display(self, frame: np.ndarray) -> None:
display_frame = frame
- if self._last_pose and self._last_pose.pose is not None:
+ if (
+ self.show_predictions_checkbox.isChecked()
+ and self._last_pose
+ and self._last_pose.pose is not None
+ ):
display_frame = self._draw_pose(frame, self._last_pose.pose)
rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB)
h, w, ch = rgb.shape
@@ -499,6 +671,19 @@ def _update_video_display(self, frame: np.ndarray) -> None:
image = QImage(rgb.data, w, h, bytes_per_line, QImage.Format.Format_RGB888)
self.video_label.setPixmap(QPixmap.fromImage(image))
+ def _apply_rotation(self, frame: np.ndarray) -> np.ndarray:
+ if self._rotation_degrees == 90:
+ return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
+ if self._rotation_degrees == 180:
+ return cv2.rotate(frame, cv2.ROTATE_180)
+ if self._rotation_degrees == 270:
+ return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
+ return frame
+
+ def _on_show_predictions_changed(self, _state: int) -> None:
+ if self._current_frame is not None:
+ self._update_video_display(self._current_frame)
+
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
for keypoint in np.asarray(pose):
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index e0e3706..3d15b1c 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -32,7 +32,7 @@ def start(self) -> None:
if self._writer is not None:
return
self._output.parent.mkdir(parents=True, exist_ok=True)
- self._writer = WriteGear(output_filename=str(self._output), logging=False, **self._options)
+ self._writer = WriteGear(output=str(self._output), logging=False, **self._options)
def write(self, frame: np.ndarray) -> None:
if self._writer is None:
From 0b0ac3308e3d70d3a0f22353ed06cdc64d356448 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Tue, 21 Oct 2025 14:57:43 +0200
Subject: [PATCH 03/26] modifyed gentl_backend
---
dlclivegui/cameras/gentl_backend.py | 380 +++++++++++++++++++++-------
dlclivegui/gui.py | 10 +-
2 files changed, 294 insertions(+), 96 deletions(-)
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 0d81294..bdbc4e8 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -1,130 +1,328 @@
-"""Generic GenTL backend implemented with Aravis."""
+"""GenTL backend implemented using the Harvesters library."""
from __future__ import annotations
-import ctypes
+import glob
+import os
import time
-from typing import Optional, Tuple
+from typing import Iterable, List, Optional, Tuple
+import cv2
import numpy as np
from .base import CameraBackend
try: # pragma: no cover - optional dependency
- import gi
-
- gi.require_version("Aravis", "0.6")
- from gi.repository import Aravis
+ from harvesters.core import Harvester
except Exception: # pragma: no cover - optional dependency
- gi = None # type: ignore
- Aravis = None # type: ignore
+ Harvester = None # type: ignore
class GenTLCameraBackend(CameraBackend):
- """Capture frames from cameras that expose a GenTL interface."""
+ """Capture frames from GenTL-compatible devices via Harvesters."""
+
+ _DEFAULT_CTI_PATTERNS: Tuple[str, ...] = (
+ r"C:\\Program Files\\The Imaging Source Europe GmbH\\IC4 GenTL Driver for USB3Vision Devices *\\bin\\*.cti",
+ r"C:\\Program Files\\The Imaging Source Europe GmbH\\TIS Grabber\\bin\\win64_x64\\*.cti",
+ r"C:\\Program Files\\The Imaging Source Europe GmbH\\TIS Camera SDK\\bin\\win64_x64\\*.cti",
+ r"C:\\Program Files (x86)\\The Imaging Source Europe GmbH\\TIS Grabber\\bin\\win64_x64\\*.cti",
+ )
def __init__(self, settings):
super().__init__(settings)
- self._camera = None
- self._stream = None
- self._payload: Optional[int] = None
+ props = settings.properties
+ self._cti_file: Optional[str] = props.get("cti_file")
+ self._serial_number: Optional[str] = props.get("serial_number") or props.get("serial")
+ self._pixel_format: str = props.get("pixel_format", "Mono8")
+ self._rotate: int = int(props.get("rotate", 0)) % 360
+ self._crop: Optional[Tuple[int, int, int, int]] = self._parse_crop(props.get("crop"))
+ self._exposure: Optional[float] = props.get("exposure")
+ self._gain: Optional[float] = props.get("gain")
+ self._timeout: float = float(props.get("timeout", 2.0))
+ self._cti_search_paths: Tuple[str, ...] = self._parse_cti_paths(props.get("cti_search_paths"))
+
+ self._harvester: Optional[Harvester] = None
+ self._acquirer = None
@classmethod
def is_available(cls) -> bool:
- return Aravis is not None
+ return Harvester is not None
def open(self) -> None:
- if Aravis is None: # pragma: no cover - optional dependency
+ if Harvester is None: # pragma: no cover - optional dependency
raise RuntimeError(
- "Aravis (python-gi bindings) are required for the GenTL backend"
+ "The 'harvesters' package is required for the GenTL backend. "
+ "Install it via 'pip install harvesters'."
)
- Aravis.update_device_list()
- num_devices = Aravis.get_n_devices()
- if num_devices == 0:
- raise RuntimeError("No GenTL cameras detected")
- device_id = self._select_device_id(num_devices)
- self._camera = Aravis.Camera.new(device_id)
- self._camera.set_exposure_time_auto(0)
- self._camera.set_gain_auto(0)
- exposure = self.settings.properties.get("exposure")
- if exposure is not None:
- self._set_exposure(float(exposure))
- crop = self.settings.properties.get("crop")
- if isinstance(crop, (list, tuple)) and len(crop) == 4:
- self._set_crop(crop)
- if self.settings.fps:
- try:
- self._camera.set_frame_rate(float(self.settings.fps))
- except Exception:
- pass
- self._stream = self._camera.create_stream()
- self._payload = self._camera.get_payload()
- self._stream.push_buffer(Aravis.Buffer.new_allocate(self._payload))
- self._camera.start_acquisition()
+
+ self._harvester = Harvester()
+ cti_file = self._cti_file or self._find_cti_file()
+ self._harvester.add_file(cti_file)
+ self._harvester.update()
+
+ if not self._harvester.device_info_list:
+ raise RuntimeError("No GenTL cameras detected via Harvesters")
+
+ serial = self._serial_number
+ index = int(self.settings.index or 0)
+ if serial:
+ available = self._available_serials()
+ matches = [s for s in available if serial in s]
+ if not matches:
+ raise RuntimeError(
+ f"Camera with serial '{serial}' not found. Available cameras: {available}"
+ )
+ serial = matches[0]
+ else:
+ device_count = len(self._harvester.device_info_list)
+ if index < 0 or index >= device_count:
+ raise RuntimeError(
+ f"Camera index {index} out of range for {device_count} GenTL device(s)"
+ )
+
+ self._acquirer = self._create_acquirer(serial, index)
+
+ remote = self._acquirer.remote_device
+ node_map = remote.node_map
+
+ self._configure_pixel_format(node_map)
+ self._configure_resolution(node_map)
+ self._configure_exposure(node_map)
+ self._configure_gain(node_map)
+ self._configure_frame_rate(node_map)
+
+ self._acquirer.start()
def read(self) -> Tuple[np.ndarray, float]:
- if self._stream is None:
- raise RuntimeError("GenTL stream not initialised")
- buffer = None
- while buffer is None:
- buffer = self._stream.try_pop_buffer()
- if buffer is None:
- time.sleep(0.01)
- frame = self._buffer_to_numpy(buffer)
- self._stream.push_buffer(buffer)
- return frame, time.time()
+ if self._acquirer is None:
+ raise RuntimeError("GenTL image acquirer not initialised")
- def close(self) -> None:
- if self._camera is not None:
+ with self._acquirer.fetch(timeout=self._timeout) as buffer:
+ component = buffer.payload.components[0]
+ channels = 3 if self._pixel_format in {"RGB8", "BGR8"} else 1
+ if channels > 1:
+ frame = component.data.reshape(
+ component.height, component.width, channels
+ ).copy()
+ else:
+ frame = component.data.reshape(component.height, component.width).copy()
+
+ frame = self._convert_frame(frame)
+ timestamp = time.time()
+ return frame, timestamp
+
+ def stop(self) -> None:
+ if self._acquirer is not None:
try:
- self._camera.stop_acquisition()
+ self._acquirer.stop()
except Exception:
pass
- self._camera = None
- self._stream = None
- self._payload = None
- def stop(self) -> None:
- if self._camera is not None:
+ def close(self) -> None:
+ if self._acquirer is not None:
try:
- self._camera.stop_acquisition()
+ self._acquirer.stop()
except Exception:
pass
+ try:
+ destroy = getattr(self._acquirer, "destroy", None)
+ if destroy is not None:
+ destroy()
+ finally:
+ self._acquirer = None
- def _select_device_id(self, num_devices: int) -> str:
- index = int(self.settings.index)
- if index < 0 or index >= num_devices:
- raise RuntimeError(
- f"Camera index {index} out of range for {num_devices} GenTL device(s)"
+ if self._harvester is not None:
+ try:
+ self._harvester.reset()
+ finally:
+ self._harvester = None
+
+ # ------------------------------------------------------------------
+ # Helpers
+ # ------------------------------------------------------------------
+
+ def _parse_cti_paths(self, value) -> Tuple[str, ...]:
+ if value is None:
+ return self._DEFAULT_CTI_PATTERNS
+ if isinstance(value, str):
+ return (value,)
+ if isinstance(value, Iterable):
+ return tuple(str(item) for item in value)
+ return self._DEFAULT_CTI_PATTERNS
+
+ def _parse_crop(self, crop) -> Optional[Tuple[int, int, int, int]]:
+ if isinstance(crop, (list, tuple)) and len(crop) == 4:
+ return tuple(int(v) for v in crop)
+ return None
+
+ def _find_cti_file(self) -> str:
+ patterns: List[str] = list(self._cti_search_paths)
+ for pattern in patterns:
+ for file_path in glob.glob(pattern):
+ if os.path.isfile(file_path):
+ return file_path
+ raise RuntimeError(
+ "Could not locate a GenTL producer (.cti) file. Set 'cti_file' in "
+ "camera.properties or provide search paths via 'cti_search_paths'."
+ )
+
+ def _available_serials(self) -> List[str]:
+ assert self._harvester is not None
+ serials: List[str] = []
+ for info in self._harvester.device_info_list:
+ serial = getattr(info, "serial_number", "")
+ if serial:
+ serials.append(serial)
+ return serials
+
+ def _create_acquirer(self, serial: Optional[str], index: int):
+ assert self._harvester is not None
+ methods = [
+ getattr(self._harvester, "create_image_acquirer", None),
+ getattr(self._harvester, "create", None),
+ ]
+ methods = [m for m in methods if m is not None]
+ errors: List[str] = []
+ device_info = None
+ if not serial:
+ device_list = self._harvester.device_info_list
+ if 0 <= index < len(device_list):
+ device_info = device_list[index]
+ for create in methods:
+ try:
+ if serial:
+ return create({"serial_number": serial})
+ except Exception as exc:
+ errors.append(f"{create.__name__} serial: {exc}")
+ for create in methods:
+ try:
+ return create(index=index)
+ except TypeError:
+ try:
+ return create(index)
+ except Exception as exc:
+ errors.append(f"{create.__name__} index positional: {exc}")
+ except Exception as exc:
+ errors.append(f"{create.__name__} index: {exc}")
+ if device_info is not None:
+ for create in methods:
+ try:
+ return create(device_info)
+ except Exception as exc:
+ errors.append(f"{create.__name__} device_info: {exc}")
+ if not serial and index == 0:
+ for create in methods:
+ try:
+ return create()
+ except Exception as exc:
+ errors.append(f"{create.__name__} default: {exc}")
+ joined = "; ".join(errors) or "no creation methods available"
+ raise RuntimeError(f"Failed to initialise GenTL image acquirer ({joined})")
+
+ def _configure_pixel_format(self, node_map) -> None:
+ try:
+ if self._pixel_format in node_map.PixelFormat.symbolics:
+ node_map.PixelFormat.value = self._pixel_format
+ except Exception:
+ pass
+
+ def _configure_resolution(self, node_map) -> None:
+ width = int(self.settings.width)
+ height = int(self.settings.height)
+ if self._rotate in (90, 270):
+ width, height = height, width
+ try:
+ node_map.Width.value = self._adjust_to_increment(
+ width, node_map.Width.min, node_map.Width.max, node_map.Width.inc
)
- return Aravis.get_device_id(index)
+ except Exception:
+ pass
+ try:
+ node_map.Height.value = self._adjust_to_increment(
+ height, node_map.Height.min, node_map.Height.max, node_map.Height.inc
+ )
+ except Exception:
+ pass
- def _set_exposure(self, exposure: float) -> None:
- if self._camera is None:
+ def _configure_exposure(self, node_map) -> None:
+ if self._exposure is None:
return
- exposure = max(0.0, min(exposure, 1.0))
- self._camera.set_exposure_time(exposure * 1e6)
+ for attr in ("ExposureAuto", "ExposureTime", "Exposure"):
+ try:
+ node = getattr(node_map, attr)
+ except AttributeError:
+ continue
+ try:
+ if attr == "ExposureAuto":
+ node.value = "Off"
+ else:
+ node.value = float(self._exposure)
+ return
+ except Exception:
+ continue
- def _set_crop(self, crop) -> None:
- if self._camera is None:
+ def _configure_gain(self, node_map) -> None:
+ if self._gain is None:
return
- left, right, top, bottom = map(int, crop)
- width = right - left
- height = bottom - top
- self._camera.set_region(left, top, width, height)
-
- def _buffer_to_numpy(self, buffer) -> np.ndarray:
- pixel_format = buffer.get_image_pixel_format()
- bits_per_pixel = (pixel_format >> 16) & 0xFF
- if bits_per_pixel == 8:
- int_pointer = ctypes.POINTER(ctypes.c_uint8)
- else:
- int_pointer = ctypes.POINTER(ctypes.c_uint16)
- addr = buffer.get_data()
- ptr = ctypes.cast(addr, int_pointer)
- frame = np.ctypeslib.as_array(ptr, (buffer.get_image_height(), buffer.get_image_width()))
- frame = frame.copy()
- if frame.ndim < 3:
- import cv2
-
- frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
- return frame
+ for attr in ("GainAuto", "Gain"):
+ try:
+ node = getattr(node_map, attr)
+ except AttributeError:
+ continue
+ try:
+ if attr == "GainAuto":
+ node.value = "Off"
+ else:
+ node.value = float(self._gain)
+ return
+ except Exception:
+ continue
+
+ def _configure_frame_rate(self, node_map) -> None:
+ if not self.settings.fps:
+ return
+ try:
+ node_map.AcquisitionFrameRateEnable.value = True
+ except Exception:
+ pass
+ try:
+ node_map.AcquisitionFrameRate.value = float(self.settings.fps)
+ except Exception:
+ pass
+
+ @staticmethod
+ def _adjust_to_increment(value: int, minimum: int, maximum: int, increment: int) -> int:
+ value = max(minimum, min(maximum, value))
+ if increment <= 0:
+ return value
+ return minimum + ((value - minimum) // increment) * increment
+
+ def _convert_frame(self, frame: np.ndarray) -> np.ndarray:
+ result = frame.astype(np.float32 if frame.dtype == np.float64 else frame.dtype)
+ if result.dtype != np.uint8:
+ max_val = np.max(result)
+ if max_val > 0:
+ result = (result / max_val * 255.0).astype(np.uint8)
+ else:
+ result = np.zeros_like(result, dtype=np.uint8)
+ if result.ndim == 2:
+ result = cv2.cvtColor(result, cv2.COLOR_GRAY2BGR)
+ elif result.ndim == 3 and result.shape[2] == 3 and self._pixel_format == "RGB8":
+ result = cv2.cvtColor(result, cv2.COLOR_RGB2BGR)
+
+ if self._rotate == 90:
+ result = cv2.rotate(result, cv2.ROTATE_90_CLOCKWISE)
+ elif self._rotate == 180:
+ result = cv2.rotate(result, cv2.ROTATE_180)
+ elif self._rotate == 270:
+ result = cv2.rotate(result, cv2.ROTATE_90_COUNTERCLOCKWISE)
+
+ if self._crop is not None:
+ top, bottom, left, right = self._crop
+ height, width = result.shape[:2]
+ top = max(0, min(height, top))
+ bottom = max(top, min(height, bottom))
+ left = max(0, min(width, left))
+ right = max(left, min(width, right))
+ result = result[top:bottom, left:right]
+
+ return result.copy()
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 4eb3971..67baded 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -31,17 +31,17 @@
QWidget,
)
-from .camera_controller import CameraController, FrameData
-from .cameras import CameraFactory
-from .config import (
+from dlclivegui.camera_controller import CameraController, FrameData
+from dlclivegui.cameras import CameraFactory
+from dlclivegui.config import (
ApplicationSettings,
CameraSettings,
DLCProcessorSettings,
RecordingSettings,
DEFAULT_CONFIG,
)
-from .dlc_processor import DLCLiveProcessor, PoseResult
-from .video_recorder import VideoRecorder
+from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult
+from dlclivegui.video_recorder import VideoRecorder
class MainWindow(QMainWindow):
From 8110085653a1e998939712459ab0961c6ce2174b Mon Sep 17 00:00:00 2001
From: Artur <35294812+arturoptophys@users.noreply.github.com>
Date: Tue, 21 Oct 2025 16:12:25 +0200
Subject: [PATCH 04/26] Improve camera control flow and recording alignment
---
dlclivegui/camera_controller.py | 18 +-
dlclivegui/cameras/base.py | 5 +
dlclivegui/cameras/factory.py | 62 ++++-
dlclivegui/cameras/opencv_backend.py | 16 ++
dlclivegui/dlc_processor.py | 20 ++
dlclivegui/gui.py | 338 +++++++++++++++++++++++----
dlclivegui/video_recorder.py | 25 +-
7 files changed, 425 insertions(+), 59 deletions(-)
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 3398566..2fb706e 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -1,12 +1,12 @@
"""Camera management for the DLC Live GUI."""
from __future__ import annotations
-import time
from dataclasses import dataclass
+from threading import Event
from typing import Optional
import numpy as np
-from PyQt6.QtCore import QMetaObject, QObject, QThread, Qt, pyqtSignal, pyqtSlot
+from PyQt6.QtCore import QObject, QThread, QMetaObject, Qt, pyqtSignal, pyqtSlot
from .cameras import CameraFactory
from .cameras.base import CameraBackend
@@ -31,12 +31,12 @@ class CameraWorker(QObject):
def __init__(self, settings: CameraSettings):
super().__init__()
self._settings = settings
- self._running = False
+ self._stop_event = Event()
self._backend: Optional[CameraBackend] = None
@pyqtSlot()
def run(self) -> None:
- self._running = True
+ self._stop_event.clear()
try:
self._backend = CameraFactory.create(self._settings)
self._backend.open()
@@ -45,7 +45,7 @@ def run(self) -> None:
self.finished.emit()
return
- while self._running:
+ while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
except Exception as exc: # pragma: no cover - device specific
@@ -63,7 +63,7 @@ def run(self) -> None:
@pyqtSlot()
def stop(self) -> None:
- self._running = False
+ self._stop_event.set()
if self._backend is not None:
try:
self._backend.stop()
@@ -106,10 +106,12 @@ def stop(self) -> None:
if not self.is_running():
return
assert self._worker is not None
+ assert self._thread is not None
QMetaObject.invokeMethod(
- self._worker, "stop", Qt.ConnectionType.QueuedConnection
+ self._worker,
+ "stop",
+ Qt.ConnectionType.QueuedConnection,
)
- assert self._thread is not None
self._thread.quit()
self._thread.wait()
diff --git a/dlclivegui/cameras/base.py b/dlclivegui/cameras/base.py
index 6ae79dc..910331c 100644
--- a/dlclivegui/cameras/base.py
+++ b/dlclivegui/cameras/base.py
@@ -33,6 +33,11 @@ def stop(self) -> None:
# Most backends do not require additional handling, but subclasses may
# override when they need to interrupt blocking reads.
+ def device_name(self) -> str:
+ """Return a human readable name for the device currently in use."""
+
+ return self.settings.name
+
@abstractmethod
def open(self) -> None:
"""Open the capture device."""
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index e9704ef..c67f7bd 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -2,12 +2,21 @@
from __future__ import annotations
import importlib
-from typing import Dict, Iterable, Tuple, Type
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Tuple, Type
from ..config import CameraSettings
from .base import CameraBackend
+@dataclass
+class DetectedCamera:
+ """Information about a camera discovered during probing."""
+
+ index: int
+ label: str
+
+
_BACKENDS: Dict[str, Tuple[str, str]] = {
"opencv": ("dlclivegui.cameras.opencv_backend", "OpenCVCameraBackend"),
"basler": ("dlclivegui.cameras.basler_backend", "BaslerCameraBackend"),
@@ -38,6 +47,57 @@ def available_backends() -> Dict[str, bool]:
availability[name] = backend_cls.is_available()
return availability
+ @staticmethod
+ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
+ """Probe ``backend`` for available cameras.
+
+ Parameters
+ ----------
+ backend:
+ The backend identifier, e.g. ``"opencv"``.
+ max_devices:
+ Upper bound for the indices that should be probed.
+
+ Returns
+ -------
+ list of :class:`DetectedCamera`
+ Sorted list of detected cameras with human readable labels.
+ """
+
+ try:
+ backend_cls = CameraFactory._resolve_backend(backend)
+ except RuntimeError:
+ return []
+ if not backend_cls.is_available():
+ return []
+
+ detected: List[DetectedCamera] = []
+ for index in range(max_devices):
+ settings = CameraSettings(
+ name=f"Probe {index}",
+ index=index,
+ width=640,
+ height=480,
+ fps=30.0,
+ backend=backend,
+ properties={},
+ )
+ backend_instance = backend_cls(settings)
+ try:
+ backend_instance.open()
+ except Exception:
+ continue
+ else:
+ label = backend_instance.device_name()
+ detected.append(DetectedCamera(index=index, label=label))
+ finally:
+ try:
+ backend_instance.close()
+ except Exception:
+ pass
+ detected.sort(key=lambda camera: camera.index)
+ return detected
+
@staticmethod
def create(settings: CameraSettings) -> CameraBackend:
"""Instantiate a backend for ``settings``."""
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index a64e3f1..8497bfa 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -39,6 +39,22 @@ def close(self) -> None:
self._capture.release()
self._capture = None
+ def stop(self) -> None:
+ if self._capture is not None:
+ self._capture.release()
+ self._capture = None
+
+ def device_name(self) -> str:
+ base_name = "OpenCV"
+ if self._capture is not None and hasattr(self._capture, "getBackendName"):
+ try:
+ backend_name = self._capture.getBackendName()
+ except Exception: # pragma: no cover - backend specific
+ backend_name = ""
+ if backend_name:
+ base_name = backend_name
+ return f"{base_name} camera #{self.settings.index}"
+
def _configure_capture(self) -> None:
if self._capture is None:
return
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 5c199b8..0e1ef3e 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -45,6 +45,18 @@ def __init__(self) -> None:
def configure(self, settings: DLCProcessorSettings) -> None:
self._settings = settings
+ def reset(self) -> None:
+ """Cancel pending work and drop the current DLCLive instance."""
+
+ with self._lock:
+ if self._pending is not None and not self._pending.done():
+ self._pending.cancel()
+ self._pending = None
+ if self._init_future is not None and not self._init_future.done():
+ self._init_future.cancel()
+ self._init_future = None
+ self._dlc = None
+
def shutdown(self) -> None:
with self._lock:
if self._pending is not None:
@@ -106,6 +118,10 @@ def _on_initialised(self, future: Future[Any]) -> None:
except Exception as exc: # pragma: no cover - runtime behaviour
LOGGER.exception("Failed to initialise DLCLive", exc_info=exc)
self.error.emit(str(exc))
+ finally:
+ with self._lock:
+ if self._init_future is future:
+ self._init_future = None
def _run_inference(self, frame: np.ndarray, timestamp: float) -> PoseResult:
if self._dlc is None:
@@ -120,4 +136,8 @@ def _on_pose_ready(self, future: Future[Any]) -> None:
LOGGER.exception("Pose inference failed", exc_info=exc)
self.error.emit(str(exc))
return
+ finally:
+ with self._lock:
+ if self._pending is future:
+ self._pending = None
self.pose_ready.emit(result)
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 4eb3971..507b199 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -24,6 +24,7 @@
QMessageBox,
QPlainTextEdit,
QPushButton,
+ QSizePolicy,
QSpinBox,
QDoubleSpinBox,
QStatusBar,
@@ -33,6 +34,7 @@
from .camera_controller import CameraController, FrameData
from .cameras import CameraFactory
+from .cameras.factory import DetectedCamera
from .config import (
ApplicationSettings,
CameraSettings,
@@ -53,8 +55,13 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._config = config or DEFAULT_CONFIG
self._config_path: Optional[Path] = None
self._current_frame: Optional[np.ndarray] = None
+ self._raw_frame: Optional[np.ndarray] = None
self._last_pose: Optional[PoseResult] = None
+ self._dlc_active: bool = False
self._video_recorder: Optional[VideoRecorder] = None
+ self._rotation_degrees: int = 0
+ self._detected_cameras: list[DetectedCamera] = []
+ self._active_camera_settings: Optional[CameraSettings] = None
self.camera_controller = CameraController()
self.dlc_processor = DLCLiveProcessor()
@@ -62,20 +69,26 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._setup_ui()
self._connect_signals()
self._apply_config(self._config)
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
# ------------------------------------------------------------------ UI
def _setup_ui(self) -> None:
central = QWidget()
- layout = QVBoxLayout(central)
+ layout = QHBoxLayout(central)
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
- layout.addWidget(self.video_label)
+ self.video_label.setSizePolicy(
+ QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding
+ )
- layout.addWidget(self._build_camera_group())
- layout.addWidget(self._build_dlc_group())
- layout.addWidget(self._build_recording_group())
+ controls_widget = QWidget()
+ controls_layout = QVBoxLayout(controls_widget)
+ controls_layout.addWidget(self._build_camera_group())
+ controls_layout.addWidget(self._build_dlc_group())
+ controls_layout.addWidget(self._build_recording_group())
button_bar = QHBoxLayout()
self.preview_button = QPushButton("Start Preview")
@@ -83,7 +96,15 @@ def _setup_ui(self) -> None:
self.stop_preview_button.setEnabled(False)
button_bar.addWidget(self.preview_button)
button_bar.addWidget(self.stop_preview_button)
- layout.addLayout(button_bar)
+ controls_layout.addLayout(button_bar)
+ controls_layout.addStretch(1)
+
+ preview_layout = QVBoxLayout()
+ preview_layout.addWidget(self.video_label)
+ preview_layout.addStretch(1)
+
+ layout.addWidget(controls_widget)
+ layout.addLayout(preview_layout, stretch=1)
self.setCentralWidget(central)
self.setStatusBar(QStatusBar())
@@ -113,10 +134,23 @@ def _build_camera_group(self) -> QGroupBox:
group = QGroupBox("Camera settings")
form = QFormLayout(group)
+ self.camera_backend = QComboBox()
+ self.camera_backend.setEditable(True)
+ availability = CameraFactory.available_backends()
+ for backend in CameraFactory.backend_names():
+ label = backend
+ if not availability.get(backend, True):
+ label = f"{backend} (unavailable)"
+ self.camera_backend.addItem(label, backend)
+ form.addRow("Backend", self.camera_backend)
+
+ index_layout = QHBoxLayout()
self.camera_index = QComboBox()
self.camera_index.setEditable(True)
- self.camera_index.addItems([str(i) for i in range(5)])
- form.addRow("Camera index", self.camera_index)
+ index_layout.addWidget(self.camera_index)
+ self.refresh_cameras_button = QPushButton("Refresh")
+ index_layout.addWidget(self.refresh_cameras_button)
+ form.addRow("Camera", index_layout)
self.camera_width = QSpinBox()
self.camera_width.setRange(1, 7680)
@@ -131,16 +165,6 @@ def _build_camera_group(self) -> QGroupBox:
self.camera_fps.setDecimals(2)
form.addRow("Frame rate", self.camera_fps)
- self.camera_backend = QComboBox()
- self.camera_backend.setEditable(True)
- availability = CameraFactory.available_backends()
- for backend in CameraFactory.backend_names():
- label = backend
- if not availability.get(backend, True):
- label = f"{backend} (unavailable)"
- self.camera_backend.addItem(label, backend)
- form.addRow("Backend", self.camera_backend)
-
self.camera_properties_edit = QPlainTextEdit()
self.camera_properties_edit.setPlaceholderText(
'{"exposure": 15000, "gain": 0.5, "serial": "123456"}'
@@ -148,6 +172,13 @@ def _build_camera_group(self) -> QGroupBox:
self.camera_properties_edit.setFixedHeight(60)
form.addRow("Advanced properties", self.camera_properties_edit)
+ self.rotation_combo = QComboBox()
+ self.rotation_combo.addItem("0° (default)", 0)
+ self.rotation_combo.addItem("90°", 90)
+ self.rotation_combo.addItem("180°", 180)
+ self.rotation_combo.addItem("270°", 270)
+ form.addRow("Rotation", self.rotation_combo)
+
return group
def _build_dlc_group(self) -> QGroupBox:
@@ -185,9 +216,18 @@ def _build_dlc_group(self) -> QGroupBox:
self.additional_options_edit.setFixedHeight(60)
form.addRow("Additional options", self.additional_options_edit)
- self.enable_dlc_checkbox = QCheckBox("Enable pose estimation")
- self.enable_dlc_checkbox.setChecked(True)
- form.addRow(self.enable_dlc_checkbox)
+ inference_buttons = QHBoxLayout()
+ self.start_inference_button = QPushButton("Start pose inference")
+ self.start_inference_button.setEnabled(False)
+ inference_buttons.addWidget(self.start_inference_button)
+ self.stop_inference_button = QPushButton("Stop pose inference")
+ self.stop_inference_button.setEnabled(False)
+ inference_buttons.addWidget(self.stop_inference_button)
+ form.addRow(inference_buttons)
+
+ self.show_predictions_checkbox = QCheckBox("Display pose predictions")
+ self.show_predictions_checkbox.setChecked(True)
+ form.addRow(self.show_predictions_checkbox)
return group
@@ -236,19 +276,29 @@ def _connect_signals(self) -> None:
self.stop_preview_button.clicked.connect(self._stop_preview)
self.start_record_button.clicked.connect(self._start_recording)
self.stop_record_button.clicked.connect(self._stop_recording)
+ self.refresh_cameras_button.clicked.connect(
+ lambda: self._refresh_camera_indices(keep_current=True)
+ )
+ self.camera_backend.currentIndexChanged.connect(self._on_backend_changed)
+ self.camera_backend.editTextChanged.connect(self._on_backend_changed)
+ self.rotation_combo.currentIndexChanged.connect(self._on_rotation_changed)
+ self.start_inference_button.clicked.connect(self._start_inference)
+ self.stop_inference_button.clicked.connect(lambda: self._stop_inference())
+ self.show_predictions_checkbox.stateChanged.connect(
+ self._on_show_predictions_changed
+ )
self.camera_controller.frame_ready.connect(self._on_frame_ready)
self.camera_controller.error.connect(self._show_error)
self.camera_controller.stopped.connect(self._on_camera_stopped)
self.dlc_processor.pose_ready.connect(self._on_pose_ready)
- self.dlc_processor.error.connect(self._show_error)
+ self.dlc_processor.error.connect(self._on_dlc_error)
self.dlc_processor.initialized.connect(self._on_dlc_initialised)
# ------------------------------------------------------------------ config
def _apply_config(self, config: ApplicationSettings) -> None:
camera = config.camera
- self.camera_index.setCurrentText(str(camera.index))
self.camera_width.setValue(int(camera.width))
self.camera_height.setValue(int(camera.height))
self.camera_fps.setValue(float(camera.fps))
@@ -258,9 +308,14 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.camera_backend.setCurrentIndex(index)
else:
self.camera_backend.setEditText(backend_name)
+ self._refresh_camera_indices(keep_current=False)
+ self._select_camera_by_index(
+ camera.index, fallback_text=camera.name or str(camera.index)
+ )
self.camera_properties_edit.setPlainText(
json.dumps(camera.properties, indent=2) if camera.properties else ""
)
+ self._active_camera_settings = None
dlc = config.dlc
self.model_path_edit.setText(dlc.model_path)
@@ -289,20 +344,14 @@ def _current_config(self) -> ApplicationSettings:
)
def _camera_settings_from_ui(self) -> CameraSettings:
- index_text = self.camera_index.currentText().strip() or "0"
- try:
- index = int(index_text)
- except ValueError:
- raise ValueError("Camera index must be an integer") from None
- backend_data = self.camera_backend.currentData()
- backend_text = (
- backend_data
- if isinstance(backend_data, str) and backend_data
- else self.camera_backend.currentText().strip()
- )
+ index = self._current_camera_index_value()
+ if index is None:
+ raise ValueError("Camera selection must provide a numeric index")
+ backend_text = self._current_backend_name()
properties = self._parse_json(self.camera_properties_edit.toPlainText())
- return CameraSettings(
- name=f"Camera {index}",
+ name_text = self.camera_index.currentText().strip()
+ settings = CameraSettings(
+ name=name_text or f"Camera {index}",
index=index,
width=self.camera_width.value(),
height=self.camera_height.value(),
@@ -310,6 +359,66 @@ def _camera_settings_from_ui(self) -> CameraSettings:
backend=backend_text or "opencv",
properties=properties,
)
+ return settings.apply_defaults()
+
+ def _current_backend_name(self) -> str:
+ backend_data = self.camera_backend.currentData()
+ if isinstance(backend_data, str) and backend_data:
+ return backend_data
+ text = self.camera_backend.currentText().strip()
+ return text or "opencv"
+
+ def _refresh_camera_indices(
+ self, *_args: object, keep_current: bool = True
+ ) -> None:
+ backend = self._current_backend_name()
+ detected = CameraFactory.detect_cameras(backend)
+ debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
+ print(
+ f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
+ )
+ self._detected_cameras = detected
+ previous_index = self._current_camera_index_value()
+ previous_text = self.camera_index.currentText()
+ self.camera_index.blockSignals(True)
+ self.camera_index.clear()
+ for camera in detected:
+ self.camera_index.addItem(camera.label, camera.index)
+ if keep_current and previous_index is not None:
+ self._select_camera_by_index(previous_index, fallback_text=previous_text)
+ elif detected:
+ self.camera_index.setCurrentIndex(0)
+ else:
+ if keep_current and previous_text:
+ self.camera_index.setEditText(previous_text)
+ else:
+ self.camera_index.setEditText("")
+ self.camera_index.blockSignals(False)
+
+ def _select_camera_by_index(
+ self, index: int, fallback_text: Optional[str] = None
+ ) -> None:
+ self.camera_index.blockSignals(True)
+ for row in range(self.camera_index.count()):
+ if self.camera_index.itemData(row) == index:
+ self.camera_index.setCurrentIndex(row)
+ break
+ else:
+ text = fallback_text if fallback_text is not None else str(index)
+ self.camera_index.setEditText(text)
+ self.camera_index.blockSignals(False)
+
+ def _current_camera_index_value(self) -> Optional[int]:
+ data = self.camera_index.currentData()
+ if isinstance(data, int):
+ return data
+ text = self.camera_index.currentText().strip()
+ if not text:
+ return None
+ try:
+ return int(text)
+ except ValueError:
+ return None
def _parse_optional_int(self, value: str) -> Optional[int]:
text = value.strip()
@@ -402,6 +511,18 @@ def _action_browse_directory(self) -> None:
if directory:
self.output_directory_edit.setText(directory)
+ def _on_backend_changed(self, *_args: object) -> None:
+ self._refresh_camera_indices(keep_current=False)
+
+ def _on_rotation_changed(self, _index: int) -> None:
+ data = self.rotation_combo.currentData()
+ self._rotation_degrees = int(data) if isinstance(data, int) else 0
+ if self._raw_frame is not None:
+ rotated = self._apply_rotation(self._raw_frame)
+ self._current_frame = rotated
+ self._last_pose = None
+ self._update_video_display(rotated)
+
# ------------------------------------------------------------------ camera control
def _start_preview(self) -> None:
try:
@@ -409,42 +530,121 @@ def _start_preview(self) -> None:
except ValueError as exc:
self._show_error(str(exc))
return
+ self._active_camera_settings = settings
self.camera_controller.start(settings)
self.preview_button.setEnabled(False)
self.stop_preview_button.setEnabled(True)
+ self._current_frame = None
+ self._raw_frame = None
+ self._last_pose = None
+ self._dlc_active = False
self.statusBar().showMessage("Camera preview started", 3000)
- if self.enable_dlc_checkbox.isChecked():
- self._configure_dlc()
- else:
- self._last_pose = None
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
def _stop_preview(self) -> None:
self.camera_controller.stop()
+ self._stop_inference(show_message=False)
self.preview_button.setEnabled(True)
self.stop_preview_button.setEnabled(False)
self._current_frame = None
+ self._raw_frame = None
self._last_pose = None
+ self._active_camera_settings = None
self.video_label.setPixmap(QPixmap())
self.video_label.setText("Camera preview not started")
self.statusBar().showMessage("Camera preview stopped", 3000)
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
def _on_camera_stopped(self) -> None:
self.preview_button.setEnabled(True)
self.stop_preview_button.setEnabled(False)
+ self._stop_inference(show_message=False)
+ self._update_inference_buttons()
+ self._active_camera_settings = None
+ self._update_camera_controls_enabled()
- def _configure_dlc(self) -> None:
+ def _configure_dlc(self) -> bool:
try:
settings = self._dlc_settings_from_ui()
except (ValueError, json.JSONDecodeError) as exc:
self._show_error(f"Invalid DLCLive settings: {exc}")
- self.enable_dlc_checkbox.setChecked(False)
- return
+ return False
+ if not settings.model_path:
+ self._show_error("Please select a DLCLive model before starting inference.")
+ return False
self.dlc_processor.configure(settings)
+ return True
+
+ def _update_inference_buttons(self) -> None:
+ preview_running = self.camera_controller.is_running()
+ self.start_inference_button.setEnabled(preview_running and not self._dlc_active)
+ self.stop_inference_button.setEnabled(preview_running and self._dlc_active)
+
+ def _update_camera_controls_enabled(self) -> None:
+ recording_active = (
+ self._video_recorder is not None and self._video_recorder.is_running
+ )
+ allow_changes = (
+ not self.camera_controller.is_running()
+ and not self._dlc_active
+ and not recording_active
+ )
+ widgets = [
+ self.camera_backend,
+ self.camera_index,
+ self.refresh_cameras_button,
+ self.camera_width,
+ self.camera_height,
+ self.camera_fps,
+ self.camera_properties_edit,
+ self.rotation_combo,
+ ]
+ for widget in widgets:
+ widget.setEnabled(allow_changes)
+
+ def _start_inference(self) -> None:
+ if self._dlc_active:
+ self.statusBar().showMessage("Pose inference already running", 3000)
+ return
+ if not self.camera_controller.is_running():
+ self._show_error(
+ "Start the camera preview before running pose inference."
+ )
+ return
+ if not self._configure_dlc():
+ self._update_inference_buttons()
+ return
+ self.dlc_processor.reset()
+ self._last_pose = None
+ self._dlc_active = True
+ self.statusBar().showMessage("Starting pose inference…", 3000)
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
+
+ def _stop_inference(self, show_message: bool = True) -> None:
+ was_active = self._dlc_active
+ self._dlc_active = False
+ self.dlc_processor.reset()
+ self._last_pose = None
+ if self._current_frame is not None:
+ self._update_video_display(self._current_frame)
+ if was_active and show_message:
+ self.statusBar().showMessage("Pose inference stopped", 3000)
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
# ------------------------------------------------------------------ recording
def _start_recording(self) -> None:
if self._video_recorder and self._video_recorder.is_running:
return
+ if not self.camera_controller.is_running():
+ self._show_error("Start the camera preview before recording.")
+ return
+ if self._current_frame is None:
+ self._show_error("Wait for the first preview frame before recording.")
+ return
try:
recording = self._recording_settings_from_ui()
except json.JSONDecodeError as exc:
@@ -453,8 +653,21 @@ def _start_recording(self) -> None:
if not recording.enabled:
self._show_error("Recording is disabled in the configuration.")
return
+ frame = self._current_frame
+ assert frame is not None
+ height, width = frame.shape[:2]
+ frame_rate = (
+ self._active_camera_settings.fps
+ if self._active_camera_settings is not None
+ else self.camera_fps.value()
+ )
output_path = recording.output_path()
- self._video_recorder = VideoRecorder(output_path, recording.options)
+ self._video_recorder = VideoRecorder(
+ output_path,
+ recording.options,
+ frame_size=(int(width), int(height)),
+ frame_rate=float(frame_rate),
+ )
try:
self._video_recorder.start()
except Exception as exc: # pragma: no cover - runtime error
@@ -464,6 +677,7 @@ def _start_recording(self) -> None:
self.start_record_button.setEnabled(False)
self.stop_record_button.setEnabled(True)
self.statusBar().showMessage(f"Recording to {output_path}", 5000)
+ self._update_camera_controls_enabled()
def _stop_recording(self) -> None:
if not self._video_recorder:
@@ -473,25 +687,42 @@ def _stop_recording(self) -> None:
self.start_record_button.setEnabled(True)
self.stop_record_button.setEnabled(False)
self.statusBar().showMessage("Recording stopped", 3000)
+ self._update_camera_controls_enabled()
# ------------------------------------------------------------------ frame handling
def _on_frame_ready(self, frame_data: FrameData) -> None:
- frame = frame_data.image
+ raw_frame = frame_data.image
+ self._raw_frame = raw_frame
+ frame = self._apply_rotation(raw_frame)
self._current_frame = frame
+ if self._active_camera_settings is not None:
+ height, width = frame.shape[:2]
+ self._active_camera_settings.width = int(width)
+ self._active_camera_settings.height = int(height)
if self._video_recorder and self._video_recorder.is_running:
self._video_recorder.write(frame)
- if self.enable_dlc_checkbox.isChecked():
+ if self._dlc_active:
self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
self._update_video_display(frame)
def _on_pose_ready(self, result: PoseResult) -> None:
+ if not self._dlc_active:
+ return
self._last_pose = result
if self._current_frame is not None:
self._update_video_display(self._current_frame)
+ def _on_dlc_error(self, message: str) -> None:
+ self._stop_inference(show_message=False)
+ self._show_error(message)
+
def _update_video_display(self, frame: np.ndarray) -> None:
display_frame = frame
- if self._last_pose and self._last_pose.pose is not None:
+ if (
+ self.show_predictions_checkbox.isChecked()
+ and self._last_pose
+ and self._last_pose.pose is not None
+ ):
display_frame = self._draw_pose(frame, self._last_pose.pose)
rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB)
h, w, ch = rgb.shape
@@ -499,6 +730,19 @@ def _update_video_display(self, frame: np.ndarray) -> None:
image = QImage(rgb.data, w, h, bytes_per_line, QImage.Format.Format_RGB888)
self.video_label.setPixmap(QPixmap.fromImage(image))
+ def _apply_rotation(self, frame: np.ndarray) -> np.ndarray:
+ if self._rotation_degrees == 90:
+ return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
+ if self._rotation_degrees == 180:
+ return cv2.rotate(frame, cv2.ROTATE_180)
+ if self._rotation_degrees == 270:
+ return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
+ return frame
+
+ def _on_show_predictions_changed(self, _state: int) -> None:
+ if self._current_frame is not None:
+ self._update_video_display(self._current_frame)
+
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
for keypoint in np.asarray(pose):
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index e0e3706..c554318 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -2,7 +2,7 @@
from __future__ import annotations
from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, Optional, Tuple
import numpy as np
@@ -15,10 +15,18 @@
class VideoRecorder:
"""Thin wrapper around :class:`vidgear.gears.WriteGear`."""
- def __init__(self, output: Path | str, options: Optional[Dict[str, Any]] = None):
+ def __init__(
+ self,
+ output: Path | str,
+ options: Optional[Dict[str, Any]] = None,
+ frame_size: Optional[Tuple[int, int]] = None,
+ frame_rate: Optional[float] = None,
+ ):
self._output = Path(output)
self._options = options or {}
self._writer: Optional[WriteGear] = None
+ self._frame_size = frame_size
+ self._frame_rate = frame_rate
@property
def is_running(self) -> bool:
@@ -31,8 +39,19 @@ def start(self) -> None:
)
if self._writer is not None:
return
+ options = dict(self._options)
+ if self._frame_size and "resolution" not in options:
+ options["resolution"] = tuple(int(x) for x in self._frame_size)
+ if self._frame_rate and "frame_rate" not in options:
+ options["frame_rate"] = float(self._frame_rate)
self._output.parent.mkdir(parents=True, exist_ok=True)
- self._writer = WriteGear(output_filename=str(self._output), logging=False, **self._options)
+ self._writer = WriteGear(output=str(self._output), logging=False, **options)
+
+ def configure_stream(
+ self, frame_size: Tuple[int, int], frame_rate: Optional[float]
+ ) -> None:
+ self._frame_size = frame_size
+ self._frame_rate = frame_rate
def write(self, frame: np.ndarray) -> None:
if self._writer is None:
From efda9d31bd3ccf78a0b66c30b7b077d4268376aa Mon Sep 17 00:00:00 2001
From: Artur <35294812+arturoptophys@users.noreply.github.com>
Date: Tue, 21 Oct 2025 17:55:13 +0200
Subject: [PATCH 05/26] Improve camera stop flow and parameter propagation
---
dlclivegui/camera_controller.py | 49 ++++++++++++++++++---------
dlclivegui/cameras/basler_backend.py | 9 +++++
dlclivegui/cameras/gentl_backend.py | 5 +++
dlclivegui/cameras/opencv_backend.py | 9 +++++
dlclivegui/gui.py | 50 ++++++++++++++++++++++------
5 files changed, 97 insertions(+), 25 deletions(-)
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 2fb706e..5ebb976 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -25,6 +25,7 @@ class CameraWorker(QObject):
"""Worker object running inside a :class:`QThread`."""
frame_captured = pyqtSignal(object)
+ started = pyqtSignal(object)
error_occurred = pyqtSignal(str)
finished = pyqtSignal()
@@ -45,6 +46,8 @@ def run(self) -> None:
self.finished.emit()
return
+ self.started.emit(self._settings)
+
while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
@@ -75,7 +78,7 @@ class CameraController(QObject):
"""High level controller that manages a camera worker thread."""
frame_ready = pyqtSignal(object)
- started = pyqtSignal(CameraSettings)
+ started = pyqtSignal(object)
stopped = pyqtSignal()
error = pyqtSignal(str)
@@ -83,40 +86,56 @@ def __init__(self) -> None:
super().__init__()
self._thread: Optional[QThread] = None
self._worker: Optional[CameraWorker] = None
+ self._pending_settings: Optional[CameraSettings] = None
def is_running(self) -> bool:
return self._thread is not None and self._thread.isRunning()
def start(self, settings: CameraSettings) -> None:
if self.is_running():
- self.stop()
- self._thread = QThread()
- self._worker = CameraWorker(settings)
- self._worker.moveToThread(self._thread)
- self._thread.started.connect(self._worker.run)
- self._worker.frame_captured.connect(self.frame_ready)
- self._worker.error_occurred.connect(self.error)
- self._worker.finished.connect(self._thread.quit)
- self._worker.finished.connect(self._worker.deleteLater)
- self._thread.finished.connect(self._cleanup)
- self._thread.start()
- self.started.emit(settings)
+ self._pending_settings = settings
+ self.stop(preserve_pending=True)
+ return
+ self._pending_settings = None
+ self._start_worker(settings)
- def stop(self) -> None:
+ def stop(self, wait: bool = False, *, preserve_pending: bool = False) -> None:
if not self.is_running():
+ if not preserve_pending:
+ self._pending_settings = None
return
assert self._worker is not None
assert self._thread is not None
+ if not preserve_pending:
+ self._pending_settings = None
QMetaObject.invokeMethod(
self._worker,
"stop",
Qt.ConnectionType.QueuedConnection,
)
self._thread.quit()
- self._thread.wait()
+ if wait:
+ self._thread.wait()
+
+ def _start_worker(self, settings: CameraSettings) -> None:
+ self._thread = QThread()
+ self._worker = CameraWorker(settings)
+ self._worker.moveToThread(self._thread)
+ self._thread.started.connect(self._worker.run)
+ self._worker.frame_captured.connect(self.frame_ready)
+ self._worker.started.connect(self.started)
+ self._worker.error_occurred.connect(self.error)
+ self._worker.finished.connect(self._thread.quit)
+ self._worker.finished.connect(self._worker.deleteLater)
+ self._thread.finished.connect(self._cleanup)
+ self._thread.start()
@pyqtSlot()
def _cleanup(self) -> None:
self._thread = None
self._worker = None
self.stopped.emit()
+ if self._pending_settings is not None:
+ pending = self._pending_settings
+ self._pending_settings = None
+ self.start(pending)
diff --git a/dlclivegui/cameras/basler_backend.py b/dlclivegui/cameras/basler_backend.py
index f9e2a15..e83185a 100644
--- a/dlclivegui/cameras/basler_backend.py
+++ b/dlclivegui/cameras/basler_backend.py
@@ -61,6 +61,15 @@ def open(self) -> None:
self._converter = pylon.ImageFormatConverter()
self._converter.OutputPixelFormat = pylon.PixelType_BGR8packed
self._converter.OutputBitAlignment = pylon.OutputBitAlignment_MsbAligned
+ try:
+ self.settings.width = int(self._camera.Width.GetValue())
+ self.settings.height = int(self._camera.Height.GetValue())
+ except Exception:
+ pass
+ try:
+ self.settings.fps = float(self._camera.ResultingFrameRateAbs.GetValue())
+ except Exception:
+ pass
def read(self) -> Tuple[np.ndarray, float]:
if self._camera is None or self._converter is None:
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 0d81294..3f6e579 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -123,6 +123,11 @@ def _buffer_to_numpy(self, buffer) -> np.ndarray:
ptr = ctypes.cast(addr, int_pointer)
frame = np.ctypeslib.as_array(ptr, (buffer.get_image_height(), buffer.get_image_width()))
frame = frame.copy()
+ try:
+ self.settings.width = int(buffer.get_image_width())
+ self.settings.height = int(buffer.get_image_height())
+ except Exception:
+ pass
if frame.ndim < 3:
import cv2
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index 8497bfa..cbadf73 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -69,6 +69,15 @@ def _configure_capture(self) -> None:
except (TypeError, ValueError):
continue
self._capture.set(prop_id, float(value))
+ actual_width = self._capture.get(cv2.CAP_PROP_FRAME_WIDTH)
+ actual_height = self._capture.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ actual_fps = self._capture.get(cv2.CAP_PROP_FPS)
+ if actual_width:
+ self.settings.width = int(actual_width)
+ if actual_height:
+ self.settings.height = int(actual_height)
+ if actual_fps:
+ self.settings.fps = float(actual_fps)
def _resolve_backend(self, backend: str | None) -> int:
if backend is None:
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 507b199..ff2408e 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -289,6 +289,7 @@ def _connect_signals(self) -> None:
)
self.camera_controller.frame_ready.connect(self._on_frame_ready)
+ self.camera_controller.started.connect(self._on_camera_started)
self.camera_controller.error.connect(self._show_error)
self.camera_controller.stopped.connect(self._on_camera_stopped)
@@ -538,15 +539,52 @@ def _start_preview(self) -> None:
self._raw_frame = None
self._last_pose = None
self._dlc_active = False
- self.statusBar().showMessage("Camera preview started", 3000)
+ self.statusBar().showMessage("Starting camera preview…", 3000)
self._update_inference_buttons()
self._update_camera_controls_enabled()
def _stop_preview(self) -> None:
+ if not self.camera_controller.is_running():
+ return
+ self.preview_button.setEnabled(False)
+ self.stop_preview_button.setEnabled(False)
+ self.start_inference_button.setEnabled(False)
+ self.stop_inference_button.setEnabled(False)
+ self.statusBar().showMessage("Stopping camera preview…", 3000)
self.camera_controller.stop()
self._stop_inference(show_message=False)
+
+ def _on_camera_started(self, settings: CameraSettings) -> None:
+ self._active_camera_settings = settings
+ self.preview_button.setEnabled(False)
+ self.stop_preview_button.setEnabled(True)
+ self.camera_width.blockSignals(True)
+ self.camera_width.setValue(int(settings.width))
+ self.camera_width.blockSignals(False)
+ self.camera_height.blockSignals(True)
+ self.camera_height.setValue(int(settings.height))
+ self.camera_height.blockSignals(False)
+ if getattr(settings, "fps", None):
+ self.camera_fps.blockSignals(True)
+ self.camera_fps.setValue(float(settings.fps))
+ self.camera_fps.blockSignals(False)
+ resolution = f"{int(settings.width)}×{int(settings.height)}"
+ if getattr(settings, "fps", None):
+ fps_text = f"{float(settings.fps):.2f} FPS"
+ else:
+ fps_text = "unknown FPS"
+ self.statusBar().showMessage(
+ f"Camera preview started: {resolution} @ {fps_text}", 5000
+ )
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
+
+ def _on_camera_stopped(self) -> None:
+ if self._video_recorder and self._video_recorder.is_running:
+ self._stop_recording()
self.preview_button.setEnabled(True)
self.stop_preview_button.setEnabled(False)
+ self._stop_inference(show_message=False)
self._current_frame = None
self._raw_frame = None
self._last_pose = None
@@ -557,14 +595,6 @@ def _stop_preview(self) -> None:
self._update_inference_buttons()
self._update_camera_controls_enabled()
- def _on_camera_stopped(self) -> None:
- self.preview_button.setEnabled(True)
- self.stop_preview_button.setEnabled(False)
- self._stop_inference(show_message=False)
- self._update_inference_buttons()
- self._active_camera_settings = None
- self._update_camera_controls_enabled()
-
def _configure_dlc(self) -> bool:
try:
settings = self._dlc_settings_from_ui()
@@ -768,7 +798,7 @@ def _show_error(self, message: str) -> None:
# ------------------------------------------------------------------ Qt overrides
def closeEvent(self, event: QCloseEvent) -> None: # pragma: no cover - GUI behaviour
if self.camera_controller.is_running():
- self.camera_controller.stop()
+ self.camera_controller.stop(wait=True)
if self._video_recorder and self._video_recorder.is_running:
self._video_recorder.stop()
self.dlc_processor.shutdown()
From f39c4c466968692a75a71e88dcf50c9c71e6eecd Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Wed, 22 Oct 2025 11:56:53 +0200
Subject: [PATCH 06/26] fixed video recordings (roughly)
---
dlclivegui/camera_controller.py | 25 +++--
dlclivegui/cameras/factory.py | 2 +
dlclivegui/cameras/gentl_backend.py | 144 +++++++++++++++++++++-------
dlclivegui/config.py | 21 +++-
dlclivegui/gui.py | 42 +++++---
dlclivegui/video_recorder.py | 46 +++++++--
6 files changed, 210 insertions(+), 70 deletions(-)
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 4965dc7..821a252 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -8,9 +8,9 @@
import numpy as np
from PyQt6.QtCore import QObject, QThread, QMetaObject, Qt, pyqtSignal, pyqtSlot
-from .cameras import CameraFactory
-from .cameras.base import CameraBackend
-from .config import CameraSettings
+from dlclivegui.cameras import CameraFactory
+from dlclivegui.cameras.base import CameraBackend
+from dlclivegui.config import CameraSettings
@dataclass
@@ -51,8 +51,15 @@ def run(self) -> None:
while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
+ except TimeoutError:
+ if self._stop_event.is_set():
+ break
+ continue
except Exception as exc: # pragma: no cover - device specific
- self.error_occurred.emit(str(exc))
+ if not self._stop_event.is_set():
+ self.error_occurred.emit(str(exc))
+ break
+ if self._stop_event.is_set():
break
self.frame_captured.emit(FrameData(frame, timestamp))
@@ -113,6 +120,7 @@ def stop(self, wait: bool = False, *, preserve_pending: bool = False) -> None:
"stop",
Qt.ConnectionType.QueuedConnection,
)
+ self._worker.stop()
self._thread.quit()
if wait:
self._thread.wait()
@@ -129,15 +137,6 @@ def _start_worker(self, settings: CameraSettings) -> None:
self._worker.finished.connect(self._worker.deleteLater)
self._thread.finished.connect(self._cleanup)
self._thread.start()
- self.started.emit(settings)
-
- def stop(self) -> None:
- if not self.is_running():
- return
- assert self._worker is not None
- self._worker.stop()
- assert self._thread is not None
- self._thread.wait()
@pyqtSlot()
def _cleanup(self) -> None:
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index c67f7bd..1dc33d8 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -89,6 +89,8 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
continue
else:
label = backend_instance.device_name()
+ if not label:
+ label = f"{backend.title()} #{index}"
detected.append(DetectedCamera(index=index, label=label))
finally:
try:
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 7a15333..dc6ce7e 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -13,8 +13,13 @@
try: # pragma: no cover - optional dependency
from harvesters.core import Harvester
+ try:
+ from harvesters.core import HarvesterTimeoutError # type: ignore
+ except Exception: # pragma: no cover - optional dependency
+ HarvesterTimeoutError = TimeoutError # type: ignore
except Exception: # pragma: no cover - optional dependency
Harvester = None # type: ignore
+ HarvesterTimeoutError = TimeoutError # type: ignore
class GenTLCameraBackend(CameraBackend):
@@ -40,8 +45,9 @@ def __init__(self, settings):
self._timeout: float = float(props.get("timeout", 2.0))
self._cti_search_paths: Tuple[str, ...] = self._parse_cti_paths(props.get("cti_search_paths"))
- self._harvester: Optional[Harvester] = None
+ self._harvester = None
self._acquirer = None
+ self._device_label: Optional[str] = None
@classmethod
def is_available(cls) -> bool:
@@ -84,6 +90,8 @@ def open(self) -> None:
remote = self._acquirer.remote_device
node_map = remote.node_map
+ self._device_label = self._resolve_device_label(node_map)
+
self._configure_pixel_format(node_map)
self._configure_resolution(node_map)
self._configure_exposure(node_map)
@@ -96,15 +104,23 @@ def read(self) -> Tuple[np.ndarray, float]:
if self._acquirer is None:
raise RuntimeError("GenTL image acquirer not initialised")
- with self._acquirer.fetch(timeout=self._timeout) as buffer:
- component = buffer.payload.components[0]
- channels = 3 if self._pixel_format in {"RGB8", "BGR8"} else 1
- if channels > 1:
- frame = component.data.reshape(
- component.height, component.width, channels
- ).copy()
- else:
- frame = component.data.reshape(component.height, component.width).copy()
+ try:
+ with self._acquirer.fetch(timeout=self._timeout) as buffer:
+ component = buffer.payload.components[0]
+ channels = 3 if self._pixel_format in {"RGB8", "BGR8"} else 1
+ array = np.asarray(component.data)
+ expected = component.height * component.width * channels
+ if array.size != expected:
+ array = np.frombuffer(bytes(component.data), dtype=array.dtype)
+ try:
+ if channels > 1:
+ frame = array.reshape(component.height, component.width, channels).copy()
+ else:
+ frame = array.reshape(component.height, component.width).copy()
+ except ValueError:
+ frame = array.copy()
+ except HarvesterTimeoutError as exc:
+ raise TimeoutError(str(exc)) from exc
frame = self._convert_frame(frame)
timestamp = time.time()
@@ -136,6 +152,8 @@ def close(self) -> None:
finally:
self._harvester = None
+ self._device_label = None
+
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
@@ -177,8 +195,8 @@ def _available_serials(self) -> List[str]:
def _create_acquirer(self, serial: Optional[str], index: int):
assert self._harvester is not None
methods = [
- getattr(self._harvester, "create_image_acquirer", None),
getattr(self._harvester, "create", None),
+ getattr(self._harvester, "create_image_acquirer", None),
]
methods = [m for m in methods if m is not None]
errors: List[str] = []
@@ -280,24 +298,86 @@ def _configure_gain(self, node_map) -> None:
def _configure_frame_rate(self, node_map) -> None:
if not self.settings.fps:
return
- left, right, top, bottom = map(int, crop)
- width = right - left
- height = bottom - top
- self._camera.set_region(left, top, width, height)
-
- def _buffer_to_numpy(self, buffer) -> np.ndarray:
- pixel_format = buffer.get_image_pixel_format()
- bits_per_pixel = (pixel_format >> 16) & 0xFF
- if bits_per_pixel == 8:
- int_pointer = ctypes.POINTER(ctypes.c_uint8)
- else:
- int_pointer = ctypes.POINTER(ctypes.c_uint16)
- addr = buffer.get_data()
- ptr = ctypes.cast(addr, int_pointer)
- frame = np.ctypeslib.as_array(ptr, (buffer.get_image_height(), buffer.get_image_width()))
- frame = frame.copy()
- if frame.ndim < 3:
- import cv2
-
- frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2RGB)
- return frame
+
+ target = float(self.settings.fps)
+ for attr in ("AcquisitionFrameRateEnable", "AcquisitionFrameRateControlEnable"):
+ try:
+ getattr(node_map, attr).value = True
+ except Exception:
+ continue
+
+ for attr in ("AcquisitionFrameRate", "ResultingFrameRate", "AcquisitionFrameRateAbs"):
+ try:
+ node = getattr(node_map, attr)
+ except AttributeError:
+ continue
+ try:
+ node.value = target
+ return
+ except Exception:
+ continue
+
+ def _convert_frame(self, frame: np.ndarray) -> np.ndarray:
+ if frame.dtype != np.uint8:
+ max_val = float(frame.max()) if frame.size else 0.0
+ scale = 255.0 / max_val if max_val > 0.0 else 1.0
+ frame = np.clip(frame * scale, 0, 255).astype(np.uint8)
+
+ if frame.ndim == 2:
+ frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
+ elif frame.ndim == 3 and frame.shape[2] == 3 and self._pixel_format == "RGB8":
+ frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+
+ if self._crop is not None:
+ top, bottom, left, right = (int(v) for v in self._crop)
+ top = max(0, top)
+ left = max(0, left)
+ bottom = bottom if bottom > 0 else frame.shape[0]
+ right = right if right > 0 else frame.shape[1]
+ bottom = min(frame.shape[0], bottom)
+ right = min(frame.shape[1], right)
+ frame = frame[top:bottom, left:right]
+
+ if self._rotate in (90, 180, 270):
+ rotations = {
+ 90: cv2.ROTATE_90_CLOCKWISE,
+ 180: cv2.ROTATE_180,
+ 270: cv2.ROTATE_90_COUNTERCLOCKWISE,
+ }
+ frame = cv2.rotate(frame, rotations[self._rotate])
+
+ return frame.copy()
+
+ def _resolve_device_label(self, node_map) -> Optional[str]:
+ candidates = [
+ ("DeviceModelName", "DeviceSerialNumber"),
+ ("DeviceDisplayName", "DeviceSerialNumber"),
+ ]
+ for name_attr, serial_attr in candidates:
+ try:
+ model = getattr(node_map, name_attr).value
+ except AttributeError:
+ continue
+ serial = None
+ try:
+ serial = getattr(node_map, serial_attr).value
+ except AttributeError:
+ pass
+ if model:
+ model_str = str(model)
+ serial_str = str(serial) if serial else None
+ return f"{model_str} ({serial_str})" if serial_str else model_str
+ return None
+
+ def _adjust_to_increment(self, value: int, minimum: int, maximum: int, increment: int) -> int:
+ value = max(minimum, min(maximum, int(value)))
+ if increment <= 0:
+ return value
+ offset = value - minimum
+ steps = offset // increment
+ return minimum + steps * increment
+
+ def device_name(self) -> str:
+ if self._device_label:
+ return self._device_label
+ return super().device_name()
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index da00c5f..d57a145 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -16,7 +16,7 @@ class CameraSettings:
width: int = 640
height: int = 480
fps: float = 30.0
- backend: str = "opencv"
+ backend: str = "gentl"
properties: Dict[str, Any] = field(default_factory=dict)
def apply_defaults(self) -> "CameraSettings":
@@ -48,7 +48,8 @@ class RecordingSettings:
directory: str = str(Path.home() / "Videos" / "deeplabcut-live")
filename: str = "session.mp4"
container: str = "mp4"
- options: Dict[str, Any] = field(default_factory=dict)
+ codec: str = "libx264"
+ crf: int = 23
def output_path(self) -> Path:
"""Return the absolute output path for recordings."""
@@ -62,6 +63,18 @@ def output_path(self) -> Path:
filename = name.with_suffix(f".{self.container}")
return directory / filename
+ def writegear_options(self, fps: float) -> Dict[str, Any]:
+ """Return compression parameters for WriteGear."""
+
+ fps_value = float(fps) if fps else 30.0
+ codec_value = (self.codec or "libx264").strip() or "libx264"
+ crf_value = int(self.crf) if self.crf is not None else 23
+ return {
+ "-input_framerate": f"{fps_value:.6f}",
+ "-vcodec": codec_value,
+ "-crf": str(crf_value),
+ }
+
@dataclass
class ApplicationSettings:
@@ -77,7 +90,9 @@ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
camera = CameraSettings(**data.get("camera", {})).apply_defaults()
dlc = DLCProcessorSettings(**data.get("dlc", {}))
- recording = RecordingSettings(**data.get("recording", {}))
+ recording_data = dict(data.get("recording", {}))
+ recording_data.pop("options", None)
+ recording = RecordingSettings(**recording_data)
return cls(camera=camera, dlc=dlc, recording=recording)
def to_dict(self) -> Dict[str, Any]:
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index ffa31ec..a6aaae8 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -254,10 +254,15 @@ def _build_recording_group(self) -> QGroupBox:
self.container_combo.addItems(["mp4", "avi", "mov"])
form.addRow("Container", self.container_combo)
- self.recording_options_edit = QPlainTextEdit()
- self.recording_options_edit.setPlaceholderText('{"compression_mode": "mp4"}')
- self.recording_options_edit.setFixedHeight(60)
- form.addRow("WriteGear options", self.recording_options_edit)
+ self.codec_combo = QComboBox()
+ self.codec_combo.addItems(["h264_nvenc", "libx264"])
+ self.codec_combo.setCurrentText("libx264")
+ form.addRow("Codec", self.codec_combo)
+
+ self.crf_spin = QSpinBox()
+ self.crf_spin.setRange(0, 51)
+ self.crf_spin.setValue(23)
+ form.addRow("CRF", self.crf_spin)
self.start_record_button = QPushButton("Start recording")
self.stop_record_button = QPushButton("Stop recording")
@@ -335,7 +340,13 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.output_directory_edit.setText(recording.directory)
self.filename_edit.setText(recording.filename)
self.container_combo.setCurrentText(recording.container)
- self.recording_options_edit.setPlainText(json.dumps(recording.options, indent=2))
+ codec_index = self.codec_combo.findText(recording.codec)
+ if codec_index >= 0:
+ self.codec_combo.setCurrentIndex(codec_index)
+ else:
+ self.codec_combo.addItem(recording.codec)
+ self.codec_combo.setCurrentIndex(self.codec_combo.count() - 1)
+ self.crf_spin.setValue(int(recording.crf))
def _current_config(self) -> ApplicationSettings:
return ApplicationSettings(
@@ -510,7 +521,8 @@ def _recording_settings_from_ui(self) -> RecordingSettings:
directory=self.output_directory_edit.text().strip(),
filename=self.filename_edit.text().strip() or "session.mp4",
container=self.container_combo.currentText().strip() or "mp4",
- options=self._parse_json(self.recording_options_edit.toPlainText()),
+ codec=self.codec_combo.currentText().strip() or "libx264",
+ crf=int(self.crf_spin.value()),
)
# ------------------------------------------------------------------ actions
@@ -701,6 +713,8 @@ def _update_camera_controls_enabled(self) -> None:
self.camera_fps,
self.camera_properties_edit,
self.rotation_combo,
+ self.codec_combo,
+ self.crf_spin,
]
for widget in widgets:
widget.setEnabled(allow_changes)
@@ -746,11 +760,7 @@ def _start_recording(self) -> None:
if self._current_frame is None:
self._show_error("Wait for the first preview frame before recording.")
return
- try:
- recording = self._recording_settings_from_ui()
- except json.JSONDecodeError as exc:
- self._show_error(f"Invalid recording options: {exc}")
- return
+ recording = self._recording_settings_from_ui()
if not recording.enabled:
self._show_error("Recording is disabled in the configuration.")
return
@@ -765,9 +775,10 @@ def _start_recording(self) -> None:
output_path = recording.output_path()
self._video_recorder = VideoRecorder(
output_path,
- recording.options,
frame_size=(int(width), int(height)),
frame_rate=float(frame_rate),
+ codec=recording.codec,
+ crf=recording.crf,
)
try:
self._video_recorder.start()
@@ -795,13 +806,18 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
raw_frame = frame_data.image
self._raw_frame = raw_frame
frame = self._apply_rotation(raw_frame)
+ frame = np.ascontiguousarray(frame)
self._current_frame = frame
if self._active_camera_settings is not None:
height, width = frame.shape[:2]
self._active_camera_settings.width = int(width)
self._active_camera_settings.height = int(height)
if self._video_recorder and self._video_recorder.is_running:
- self._video_recorder.write(frame)
+ try:
+ self._video_recorder.write(frame)
+ except RuntimeError as exc:
+ self._show_error(str(exc))
+ self._stop_recording()
if self._dlc_active:
self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
self._update_video_display(frame)
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index c554318..ff52fdb 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -18,15 +18,17 @@ class VideoRecorder:
def __init__(
self,
output: Path | str,
- options: Optional[Dict[str, Any]] = None,
frame_size: Optional[Tuple[int, int]] = None,
frame_rate: Optional[float] = None,
+ codec: str = "libx264",
+ crf: int = 23,
):
self._output = Path(output)
- self._options = options or {}
self._writer: Optional[WriteGear] = None
self._frame_size = frame_size
self._frame_rate = frame_rate
+ self._codec = codec
+ self._crf = int(crf)
@property
def is_running(self) -> bool:
@@ -39,13 +41,19 @@ def start(self) -> None:
)
if self._writer is not None:
return
- options = dict(self._options)
- if self._frame_size and "resolution" not in options:
- options["resolution"] = tuple(int(x) for x in self._frame_size)
- if self._frame_rate and "frame_rate" not in options:
- options["frame_rate"] = float(self._frame_rate)
+ fps_value = float(self._frame_rate) if self._frame_rate else 30.0
+
+ writer_kwargs: Dict[str, Any] = {
+ "compression_mode": True,
+ "logging": True,
+ "-input_framerate": fps_value,
+ "-vcodec": (self._codec or "libx264").strip() or "libx264",
+ "-crf": int(self._crf),
+ }
+ # TODO deal with pixel format
+
self._output.parent.mkdir(parents=True, exist_ok=True)
- self._writer = WriteGear(output=str(self._output), logging=False, **options)
+ self._writer = WriteGear(output=str(self._output), **writer_kwargs)
def configure_stream(
self, frame_size: Tuple[int, int], frame_rate: Optional[float]
@@ -56,7 +64,27 @@ def configure_stream(
def write(self, frame: np.ndarray) -> None:
if self._writer is None:
return
- self._writer.write(frame)
+ if frame.dtype != np.uint8:
+ frame_float = frame.astype(np.float32, copy=False)
+ max_val = float(frame_float.max()) if frame_float.size else 0.0
+ scale = 1.0
+ if max_val > 0:
+ scale = 255.0 / max_val if max_val > 255.0 else (255.0 if max_val <= 1.0 else 1.0)
+ frame = np.clip(frame_float * scale, 0.0, 255.0).astype(np.uint8)
+ if frame.ndim == 2:
+ frame = np.repeat(frame[:, :, None], 3, axis=2)
+ frame = np.ascontiguousarray(frame)
+ try:
+ self._writer.write(frame)
+ except OSError as exc:
+ writer = self._writer
+ self._writer = None
+ if writer is not None:
+ try:
+ writer.close()
+ except Exception:
+ pass
+ raise RuntimeError(f"Video encoding failed: {exc}") from exc
def stop(self) -> None:
if self._writer is None:
From 7e7e3b6f8e6218cd40e3410aefeac487aa0b70b2 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 23 Oct 2025 14:45:00 +0200
Subject: [PATCH 07/26] updates
---
dlclivegui/config.py | 17 +-
dlclivegui/dlc_processor.py | 282 ++++++++++++++++++++----------
dlclivegui/gui.py | 324 +++++++++++++++++++++++++++--------
dlclivegui/video_recorder.py | 212 +++++++++++++++++++++--
setup.py | 4 +-
5 files changed, 654 insertions(+), 185 deletions(-)
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index d57a145..72e3f80 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -15,8 +15,10 @@ class CameraSettings:
index: int = 0
width: int = 640
height: int = 480
- fps: float = 30.0
+ fps: float = 25.0
backend: str = "gentl"
+ exposure: int = 500 # 0 = auto, otherwise microseconds
+ gain: float = 10 # 0.0 = auto, otherwise gain value
properties: Dict[str, Any] = field(default_factory=dict)
def apply_defaults(self) -> "CameraSettings":
@@ -25,6 +27,8 @@ def apply_defaults(self) -> "CameraSettings":
self.width = int(self.width) if self.width else 640
self.height = int(self.height) if self.height else 480
self.fps = float(self.fps) if self.fps else 30.0
+ self.exposure = int(self.exposure) if self.exposure else 0
+ self.gain = float(self.gain) if self.gain else 0.0
return self
@@ -33,11 +37,8 @@ class DLCProcessorSettings:
"""Configuration for DLCLive processing."""
model_path: str = ""
- shuffle: Optional[int] = None
- trainingsetindex: Optional[int] = None
- processor: str = "cpu"
- processor_args: Dict[str, Any] = field(default_factory=dict)
additional_options: Dict[str, Any] = field(default_factory=dict)
+ model_type: Optional[str] = "base"
@dataclass
@@ -89,7 +90,11 @@ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
"""Create an :class:`ApplicationSettings` from a dictionary."""
camera = CameraSettings(**data.get("camera", {})).apply_defaults()
- dlc = DLCProcessorSettings(**data.get("dlc", {}))
+ dlc_data = dict(data.get("dlc", {}))
+ dlc = DLCProcessorSettings(
+ model_path=str(dlc_data.get("model_path", "")),
+ additional_options=dict(dlc_data.get("additional_options", {})),
+ )
recording_data = dict(data.get("recording", {}))
recording_data.pop("options", None)
recording = RecordingSettings(**recording_data)
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 0e1ef3e..201f53c 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -2,15 +2,17 @@
from __future__ import annotations
import logging
+import queue
import threading
-from concurrent.futures import Future, ThreadPoolExecutor
-from dataclasses import dataclass
+import time
+from collections import deque
+from dataclasses import dataclass, field
from typing import Any, Optional
import numpy as np
from PyQt6.QtCore import QObject, pyqtSignal
-from .config import DLCProcessorSettings
+from dlclivegui.config import DLCProcessorSettings
LOGGER = logging.getLogger(__name__)
@@ -26,8 +28,23 @@ class PoseResult:
timestamp: float
+@dataclass
+class ProcessorStats:
+ """Statistics for DLC processor performance."""
+ frames_enqueued: int = 0
+ frames_processed: int = 0
+ frames_dropped: int = 0
+ queue_size: int = 0
+ processing_fps: float = 0.0
+ average_latency: float = 0.0
+ last_latency: float = 0.0
+
+
+_SENTINEL = object()
+
+
class DLCLiveProcessor(QObject):
- """Background pose estimation using DLCLive."""
+ """Background pose estimation using DLCLive with queue-based threading."""
pose_ready = pyqtSignal(object)
error = pyqtSignal(str)
@@ -36,108 +53,187 @@ class DLCLiveProcessor(QObject):
def __init__(self) -> None:
super().__init__()
self._settings = DLCProcessorSettings()
- self._executor = ThreadPoolExecutor(max_workers=1)
- self._dlc: Optional[DLCLive] = None
- self._init_future: Optional[Future[Any]] = None
- self._pending: Optional[Future[Any]] = None
- self._lock = threading.Lock()
+ self._dlc: Optional[Any] = None
+ self._queue: Optional[queue.Queue[Any]] = None
+ self._worker_thread: Optional[threading.Thread] = None
+ self._stop_event = threading.Event()
+ self._initialized = False
+
+ # Statistics tracking
+ self._frames_enqueued = 0
+ self._frames_processed = 0
+ self._frames_dropped = 0
+ self._latencies: deque[float] = deque(maxlen=60)
+ self._processing_times: deque[float] = deque(maxlen=60)
+ self._stats_lock = threading.Lock()
def configure(self, settings: DLCProcessorSettings) -> None:
self._settings = settings
def reset(self) -> None:
- """Cancel pending work and drop the current DLCLive instance."""
-
- with self._lock:
- if self._pending is not None and not self._pending.done():
- self._pending.cancel()
- self._pending = None
- if self._init_future is not None and not self._init_future.done():
- self._init_future.cancel()
- self._init_future = None
- self._dlc = None
+ """Stop the worker thread and drop the current DLCLive instance."""
+ self._stop_worker()
+ self._dlc = None
+ self._initialized = False
+ with self._stats_lock:
+ self._frames_enqueued = 0
+ self._frames_processed = 0
+ self._frames_dropped = 0
+ self._latencies.clear()
+ self._processing_times.clear()
def shutdown(self) -> None:
- with self._lock:
- if self._pending is not None:
- self._pending.cancel()
- self._pending = None
- if self._init_future is not None:
- self._init_future.cancel()
- self._init_future = None
- self._executor.shutdown(wait=False, cancel_futures=True)
+ self._stop_worker()
self._dlc = None
+ self._initialized = False
def enqueue_frame(self, frame: np.ndarray, timestamp: float) -> None:
- with self._lock:
- if self._dlc is None and self._init_future is None:
- self._init_future = self._executor.submit(
- self._initialise_model, frame.copy(), timestamp
- )
- self._init_future.add_done_callback(self._on_initialised)
- return
- if self._dlc is None:
- return
- if self._pending is not None and not self._pending.done():
- return
- self._pending = self._executor.submit(
- self._run_inference, frame.copy(), timestamp
+ if not self._initialized and self._worker_thread is None:
+ # Start worker thread with initialization
+ self._start_worker(frame.copy(), timestamp)
+ return
+
+ if self._queue is not None:
+ try:
+ # Non-blocking put - drop frame if queue is full
+ self._queue.put_nowait((frame.copy(), timestamp, time.perf_counter()))
+ with self._stats_lock:
+ self._frames_enqueued += 1
+ except queue.Full:
+ LOGGER.debug("DLC queue full, dropping frame")
+ with self._stats_lock:
+ self._frames_dropped += 1
+
+ def get_stats(self) -> ProcessorStats:
+ """Get current processing statistics."""
+ queue_size = self._queue.qsize() if self._queue is not None else 0
+
+ with self._stats_lock:
+ avg_latency = (
+ sum(self._latencies) / len(self._latencies)
+ if self._latencies
+ else 0.0
)
- self._pending.add_done_callback(self._on_pose_ready)
-
- def _initialise_model(self, frame: np.ndarray, timestamp: float) -> bool:
- if DLCLive is None:
- raise RuntimeError(
- "The 'dlclive' package is required for pose estimation. Install it to enable DLCLive support."
+ last_latency = self._latencies[-1] if self._latencies else 0.0
+
+ # Compute processing FPS from processing times
+ if len(self._processing_times) >= 2:
+ duration = self._processing_times[-1] - self._processing_times[0]
+ processing_fps = (len(self._processing_times) - 1) / duration if duration > 0 else 0.0
+ else:
+ processing_fps = 0.0
+
+ return ProcessorStats(
+ frames_enqueued=self._frames_enqueued,
+ frames_processed=self._frames_processed,
+ frames_dropped=self._frames_dropped,
+ queue_size=queue_size,
+ processing_fps=processing_fps,
+ average_latency=avg_latency,
+ last_latency=last_latency,
)
- if not self._settings.model_path:
- raise RuntimeError("No DLCLive model path configured.")
- options = {
- "model_path": self._settings.model_path,
- "processor": self._settings.processor,
- }
- options.update(self._settings.additional_options)
- if self._settings.shuffle is not None:
- options["shuffle"] = self._settings.shuffle
- if self._settings.trainingsetindex is not None:
- options["trainingsetindex"] = self._settings.trainingsetindex
- if self._settings.processor_args:
- options["processor_config"] = {
- "object": self._settings.processor,
- **self._settings.processor_args,
- }
- model = DLCLive(**options)
- model.init_inference(frame, frame_time=timestamp, record=False)
- self._dlc = model
- return True
- def _on_initialised(self, future: Future[Any]) -> None:
- try:
- result = future.result()
- self.initialized.emit(bool(result))
- except Exception as exc: # pragma: no cover - runtime behaviour
- LOGGER.exception("Failed to initialise DLCLive", exc_info=exc)
- self.error.emit(str(exc))
- finally:
- with self._lock:
- if self._init_future is future:
- self._init_future = None
-
- def _run_inference(self, frame: np.ndarray, timestamp: float) -> PoseResult:
- if self._dlc is None:
- raise RuntimeError("DLCLive model not initialised")
- pose = self._dlc.get_pose(frame, frame_time=timestamp, record=False)
- return PoseResult(pose=pose, timestamp=timestamp)
-
- def _on_pose_ready(self, future: Future[Any]) -> None:
+ def _start_worker(self, init_frame: np.ndarray, init_timestamp: float) -> None:
+ if self._worker_thread is not None and self._worker_thread.is_alive():
+ return
+
+ self._queue = queue.Queue(maxsize=5)
+ self._stop_event.clear()
+ self._worker_thread = threading.Thread(
+ target=self._worker_loop,
+ args=(init_frame, init_timestamp),
+ name="DLCLiveWorker",
+ daemon=True,
+ )
+ self._worker_thread.start()
+
+ def _stop_worker(self) -> None:
+ if self._worker_thread is None:
+ return
+
+ self._stop_event.set()
+ if self._queue is not None:
+ try:
+ self._queue.put_nowait(_SENTINEL)
+ except queue.Full:
+ pass
+
+ self._worker_thread.join(timeout=2.0)
+ if self._worker_thread.is_alive():
+ LOGGER.warning("DLC worker thread did not terminate cleanly")
+
+ self._worker_thread = None
+ self._queue = None
+
+ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
try:
- result = future.result()
- except Exception as exc: # pragma: no cover - runtime behaviour
- LOGGER.exception("Pose inference failed", exc_info=exc)
+ # Initialize model
+ if DLCLive is None:
+ raise RuntimeError(
+ "The 'dlclive' package is required for pose estimation."
+ )
+ if not self._settings.model_path:
+ raise RuntimeError("No DLCLive model path configured.")
+
+ options = {
+ "model_path": self._settings.model_path,
+ "model_type": self._settings.model_type,
+ "processor": None,
+ "dynamic": [False,0.5,10],
+ "resize": 1.0,
+ }
+ self._dlc = DLCLive(**options)
+ self._dlc.init_inference(init_frame)
+ self._initialized = True
+ self.initialized.emit(True)
+ LOGGER.info("DLCLive model initialized successfully")
+
+ # Process the initialization frame
+ enqueue_time = time.perf_counter()
+ pose = self._dlc.get_pose(init_frame, frame_time=init_timestamp)
+ process_time = time.perf_counter()
+
+ with self._stats_lock:
+ self._frames_enqueued += 1
+ self._frames_processed += 1
+ self._processing_times.append(process_time)
+
+ self.pose_ready.emit(PoseResult(pose=pose, timestamp=init_timestamp))
+
+ except Exception as exc:
+ LOGGER.exception("Failed to initialize DLCLive", exc_info=exc)
self.error.emit(str(exc))
+ self.initialized.emit(False)
return
- finally:
- with self._lock:
- if self._pending is future:
- self._pending = None
- self.pose_ready.emit(result)
+
+ # Main processing loop
+ while not self._stop_event.is_set():
+ try:
+ item = self._queue.get(timeout=0.1)
+ except queue.Empty:
+ continue
+
+ if item is _SENTINEL:
+ break
+
+ frame, timestamp, enqueue_time = item
+ try:
+ start_process = time.perf_counter()
+ pose = self._dlc.get_pose(frame, frame_time=timestamp)
+ end_process = time.perf_counter()
+
+ latency = end_process - enqueue_time
+
+ with self._stats_lock:
+ self._frames_processed += 1
+ self._latencies.append(latency)
+ self._processing_times.append(end_process)
+
+ self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
+ except Exception as exc:
+ LOGGER.exception("Pose inference failed", exc_info=exc)
+ self.error.emit(str(exc))
+ finally:
+ self._queue.task_done()
+
+ LOGGER.info("DLC worker thread exiting")
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index a6aaae8..0328222 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -1,14 +1,18 @@
"""PyQt6 based GUI for DeepLabCut Live."""
from __future__ import annotations
+import os
import json
import sys
+import time
+import logging
+from collections import deque
from pathlib import Path
from typing import Optional
import cv2
import numpy as np
-from PyQt6.QtCore import Qt
+from PyQt6.QtCore import Qt, QTimer
from PyQt6.QtGui import QAction, QCloseEvent, QImage, QPixmap
from PyQt6.QtWidgets import (
QApplication,
@@ -42,10 +46,14 @@
RecordingSettings,
DEFAULT_CONFIG,
)
-from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult
-from dlclivegui.video_recorder import VideoRecorder
+from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult, ProcessorStats
+from dlclivegui.video_recorder import RecorderStats, VideoRecorder
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+logging.basicConfig(level=logging.INFO)
+
+PATH2MODELS = "C:\\Users\\User\\Repos\\DeepLabCut-live-GUI\\models"
class MainWindow(QMainWindow):
"""Main application window."""
@@ -62,6 +70,12 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._rotation_degrees: int = 0
self._detected_cameras: list[DetectedCamera] = []
self._active_camera_settings: Optional[CameraSettings] = None
+ self._camera_frame_times: deque[float] = deque(maxlen=240)
+ self._last_drop_warning = 0.0
+ self._last_recorder_summary = "Recorder idle"
+ self._display_interval = 1.0 / 25.0
+ self._last_display_time = 0.0
+ self._dlc_initialized = False
self.camera_controller = CameraController()
self.dlc_processor = DLCLiveProcessor()
@@ -71,6 +85,11 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._apply_config(self._config)
self._update_inference_buttons()
self._update_camera_controls_enabled()
+ self._metrics_timer = QTimer(self)
+ self._metrics_timer.setInterval(500)
+ self._metrics_timer.timeout.connect(self._update_metrics)
+ self._metrics_timer.start()
+ self._update_metrics()
# ------------------------------------------------------------------ UI
def _setup_ui(self) -> None:
@@ -165,9 +184,23 @@ def _build_camera_group(self) -> QGroupBox:
self.camera_fps.setDecimals(2)
form.addRow("Frame rate", self.camera_fps)
+ self.camera_exposure = QSpinBox()
+ self.camera_exposure.setRange(0, 1000000)
+ self.camera_exposure.setValue(0)
+ self.camera_exposure.setSpecialValueText("Auto")
+ self.camera_exposure.setSuffix(" μs")
+ form.addRow("Exposure", self.camera_exposure)
+
+ self.camera_gain = QDoubleSpinBox()
+ self.camera_gain.setRange(0.0, 100.0)
+ self.camera_gain.setValue(0.0)
+ self.camera_gain.setSpecialValueText("Auto")
+ self.camera_gain.setDecimals(2)
+ form.addRow("Gain", self.camera_gain)
+
self.camera_properties_edit = QPlainTextEdit()
self.camera_properties_edit.setPlaceholderText(
- '{"exposure": 15000, "gain": 0.5, "serial": "123456"}'
+ '{"other_property": "value"}'
)
self.camera_properties_edit.setFixedHeight(60)
form.addRow("Advanced properties", self.camera_properties_edit)
@@ -179,6 +212,9 @@ def _build_camera_group(self) -> QGroupBox:
self.rotation_combo.addItem("270°", 270)
form.addRow("Rotation", self.rotation_combo)
+ self.camera_stats_label = QLabel("Camera idle")
+ form.addRow("Throughput", self.camera_stats_label)
+
return group
def _build_dlc_group(self) -> QGroupBox:
@@ -187,32 +223,21 @@ def _build_dlc_group(self) -> QGroupBox:
path_layout = QHBoxLayout()
self.model_path_edit = QLineEdit()
+ self.model_path_edit.setPlaceholderText("/path/to/exported/model")
path_layout.addWidget(self.model_path_edit)
browse_model = QPushButton("Browse…")
browse_model.clicked.connect(self._action_browse_model)
path_layout.addWidget(browse_model)
- form.addRow("Model path", path_layout)
-
- self.shuffle_edit = QLineEdit()
- self.shuffle_edit.setPlaceholderText("Optional integer")
- form.addRow("Shuffle", self.shuffle_edit)
-
- self.training_edit = QLineEdit()
- self.training_edit.setPlaceholderText("Optional integer")
- form.addRow("Training set index", self.training_edit)
+ form.addRow("Model directory", path_layout)
- self.processor_combo = QComboBox()
- self.processor_combo.setEditable(True)
- self.processor_combo.addItems(["cpu", "gpu", "tensorrt"])
- form.addRow("Processor", self.processor_combo)
-
- self.processor_args_edit = QPlainTextEdit()
- self.processor_args_edit.setPlaceholderText('{"device": 0}')
- self.processor_args_edit.setFixedHeight(60)
- form.addRow("Processor args", self.processor_args_edit)
+ self.model_type_combo = QComboBox()
+ self.model_type_combo.addItem("Base (TensorFlow)", "base")
+ self.model_type_combo.addItem("PyTorch", "pytorch")
+ self.model_type_combo.setCurrentIndex(0) # Default to base
+ form.addRow("Model type", self.model_type_combo)
self.additional_options_edit = QPlainTextEdit()
- self.additional_options_edit.setPlaceholderText('{"allow_growth": true}')
+ self.additional_options_edit.setPlaceholderText('')
self.additional_options_edit.setFixedHeight(60)
form.addRow("Additional options", self.additional_options_edit)
@@ -229,6 +254,10 @@ def _build_dlc_group(self) -> QGroupBox:
self.show_predictions_checkbox.setChecked(True)
form.addRow(self.show_predictions_checkbox)
+ self.dlc_stats_label = QLabel("DLC processor idle")
+ self.dlc_stats_label.setWordWrap(True)
+ form.addRow("Performance", self.dlc_stats_label)
+
return group
def _build_recording_group(self) -> QGroupBox:
@@ -256,7 +285,7 @@ def _build_recording_group(self) -> QGroupBox:
self.codec_combo = QComboBox()
self.codec_combo.addItems(["h264_nvenc", "libx264"])
- self.codec_combo.setCurrentText("libx264")
+ self.codec_combo.setCurrentText("h264_nvenc")
form.addRow("Codec", self.codec_combo)
self.crf_spin = QSpinBox()
@@ -273,6 +302,10 @@ def _build_recording_group(self) -> QGroupBox:
buttons.addWidget(self.stop_record_button)
form.addRow(buttons)
+ self.recording_stats_label = QLabel(self._last_recorder_summary)
+ self.recording_stats_label.setWordWrap(True)
+ form.addRow("Performance", self.recording_stats_label)
+
return group
# ------------------------------------------------------------------ signals
@@ -308,6 +341,11 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.camera_width.setValue(int(camera.width))
self.camera_height.setValue(int(camera.height))
self.camera_fps.setValue(float(camera.fps))
+
+ # Set exposure and gain from config
+ self.camera_exposure.setValue(int(camera.exposure))
+ self.camera_gain.setValue(float(camera.gain))
+
backend_name = camera.backend or "opencv"
index = self.camera_backend.findData(backend_name)
if index >= 0:
@@ -318,19 +356,23 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self._select_camera_by_index(
camera.index, fallback_text=camera.name or str(camera.index)
)
+
+ # Set advanced properties (exposure and gain are now separate fields)
self.camera_properties_edit.setPlainText(
json.dumps(camera.properties, indent=2) if camera.properties else ""
)
+
self._active_camera_settings = None
dlc = config.dlc
self.model_path_edit.setText(dlc.model_path)
- self.shuffle_edit.setText("" if dlc.shuffle is None else str(dlc.shuffle))
- self.training_edit.setText(
- "" if dlc.trainingsetindex is None else str(dlc.trainingsetindex)
- )
- self.processor_combo.setCurrentText(dlc.processor or "cpu")
- self.processor_args_edit.setPlainText(json.dumps(dlc.processor_args, indent=2))
+
+ # Set model type
+ model_type = dlc.model_type or "base"
+ model_type_index = self.model_type_combo.findData(model_type)
+ if model_type_index >= 0:
+ self.model_type_combo.setCurrentIndex(model_type_index)
+
self.additional_options_edit.setPlainText(
json.dumps(dlc.additional_options, indent=2)
)
@@ -361,6 +403,17 @@ def _camera_settings_from_ui(self) -> CameraSettings:
raise ValueError("Camera selection must provide a numeric index")
backend_text = self._current_backend_name()
properties = self._parse_json(self.camera_properties_edit.toPlainText())
+
+ # Get exposure and gain from explicit UI fields
+ exposure = self.camera_exposure.value()
+ gain = self.camera_gain.value()
+
+ # Also add to properties dict for backward compatibility with camera backends
+ if exposure > 0:
+ properties["exposure"] = exposure
+ if gain > 0.0:
+ properties["gain"] = gain
+
name_text = self.camera_index.currentText().strip()
settings = CameraSettings(
name=name_text or f"Camera {index}",
@@ -369,6 +422,8 @@ def _camera_settings_from_ui(self) -> CameraSettings:
height=self.camera_height.value(),
fps=self.camera_fps.value(),
backend=backend_text or "opencv",
+ exposure=exposure,
+ gain=gain,
properties=properties,
)
return settings.apply_defaults()
@@ -491,12 +546,6 @@ def _current_camera_index_value(self) -> Optional[int]:
except ValueError:
return None
- def _parse_optional_int(self, value: str) -> Optional[int]:
- text = value.strip()
- if not text:
- return None
- return int(text)
-
def _parse_json(self, value: str) -> dict:
text = value.strip()
if not text:
@@ -504,12 +553,13 @@ def _parse_json(self, value: str) -> dict:
return json.loads(text)
def _dlc_settings_from_ui(self) -> DLCProcessorSettings:
+ model_type = self.model_type_combo.currentData()
+ if not isinstance(model_type, str):
+ model_type = "base"
+
return DLCProcessorSettings(
model_path=self.model_path_edit.text().strip(),
- shuffle=self._parse_optional_int(self.shuffle_edit.text()),
- trainingsetindex=self._parse_optional_int(self.training_edit.text()),
- processor=self.processor_combo.currentText().strip() or "cpu",
- processor_args=self._parse_json(self.processor_args_edit.toPlainText()),
+ model_type=model_type,
additional_options=self._parse_json(
self.additional_options_edit.toPlainText()
),
@@ -570,11 +620,11 @@ def _save_config_to_path(self, path: Path) -> None:
self.statusBar().showMessage(f"Saved configuration to {path}", 5000)
def _action_browse_model(self) -> None:
- file_name, _ = QFileDialog.getOpenFileName(
- self, "Select DLCLive model", str(Path.home()), "All files (*.*)"
+ directory = QFileDialog.getExistingDirectory(
+ self, "Select DLCLive model directory", PATH2MODELS
)
- if file_name:
- self.model_path_edit.setText(file_name)
+ if directory:
+ self.model_path_edit.setText(directory)
def _action_browse_directory(self) -> None:
directory = QFileDialog.getExistingDirectory(
@@ -593,19 +643,7 @@ def _on_rotation_changed(self, _index: int) -> None:
rotated = self._apply_rotation(self._raw_frame)
self._current_frame = rotated
self._last_pose = None
- self._update_video_display(rotated)
-
- def _on_backend_changed(self, *_args: object) -> None:
- self._refresh_camera_indices(keep_current=False)
-
- def _on_rotation_changed(self, _index: int) -> None:
- data = self.rotation_combo.currentData()
- self._rotation_degrees = int(data) if isinstance(data, int) else 0
- if self._raw_frame is not None:
- rotated = self._apply_rotation(self._raw_frame)
- self._current_frame = rotated
- self._last_pose = None
- self._update_video_display(rotated)
+ self._display_frame(rotated, force=False)
# ------------------------------------------------------------------ camera control
def _start_preview(self) -> None:
@@ -622,6 +660,10 @@ def _start_preview(self) -> None:
self._raw_frame = None
self._last_pose = None
self._dlc_active = False
+ self._camera_frame_times.clear()
+ self._last_display_time = 0.0
+ if hasattr(self, "camera_stats_label"):
+ self.camera_stats_label.setText("Camera starting…")
self.statusBar().showMessage("Starting camera preview…", 3000)
self._update_inference_buttons()
self._update_camera_controls_enabled()
@@ -636,6 +678,10 @@ def _stop_preview(self) -> None:
self.statusBar().showMessage("Stopping camera preview…", 3000)
self.camera_controller.stop()
self._stop_inference(show_message=False)
+ self._camera_frame_times.clear()
+ self._last_display_time = 0.0
+ if hasattr(self, "camera_stats_label"):
+ self.camera_stats_label.setText("Camera idle")
def _on_camera_started(self, settings: CameraSettings) -> None:
self._active_camera_settings = settings
@@ -675,6 +721,10 @@ def _on_camera_stopped(self) -> None:
self.video_label.setPixmap(QPixmap())
self.video_label.setText("Camera preview not started")
self.statusBar().showMessage("Camera preview stopped", 3000)
+ self._camera_frame_times.clear()
+ self._last_display_time = 0.0
+ if hasattr(self, "camera_stats_label"):
+ self.camera_stats_label.setText("Camera idle")
self._update_inference_buttons()
self._update_camera_controls_enabled()
@@ -711,6 +761,8 @@ def _update_camera_controls_enabled(self) -> None:
self.camera_width,
self.camera_height,
self.camera_fps,
+ self.camera_exposure,
+ self.camera_gain,
self.camera_properties_edit,
self.rotation_combo,
self.codec_combo,
@@ -719,6 +771,90 @@ def _update_camera_controls_enabled(self) -> None:
for widget in widgets:
widget.setEnabled(allow_changes)
+ def _track_camera_frame(self) -> None:
+ now = time.perf_counter()
+ self._camera_frame_times.append(now)
+ window_seconds = 5.0
+ while self._camera_frame_times and now - self._camera_frame_times[0] > window_seconds:
+ self._camera_frame_times.popleft()
+
+ def _display_frame(self, frame: np.ndarray, *, force: bool = False) -> None:
+ if frame is None:
+ return
+ now = time.perf_counter()
+ if not force and (now - self._last_display_time) < self._display_interval:
+ return
+ self._last_display_time = now
+ self._update_video_display(frame)
+
+ def _compute_fps(self, times: deque[float]) -> float:
+ if len(times) < 2:
+ return 0.0
+ duration = times[-1] - times[0]
+ if duration <= 0:
+ return 0.0
+ return (len(times) - 1) / duration
+
+ def _format_recorder_stats(self, stats: RecorderStats) -> str:
+ latency_ms = stats.last_latency * 1000.0
+ avg_ms = stats.average_latency * 1000.0
+ buffer_ms = stats.buffer_seconds * 1000.0
+ write_fps = stats.write_fps
+ enqueue = stats.frames_enqueued
+ written = stats.frames_written
+ dropped = stats.dropped_frames
+ return (
+ f"{written}/{enqueue} frames | write {write_fps:.1f} fps | "
+ f"latency {latency_ms:.1f} ms (avg {avg_ms:.1f} ms) | "
+ f"queue {stats.queue_size} (~{buffer_ms:.0f} ms) | dropped {dropped}"
+ )
+
+ def _format_dlc_stats(self, stats: ProcessorStats) -> str:
+ """Format DLC processor statistics for display."""
+ latency_ms = stats.last_latency * 1000.0
+ avg_ms = stats.average_latency * 1000.0
+ processing_fps = stats.processing_fps
+ enqueue = stats.frames_enqueued
+ processed = stats.frames_processed
+ dropped = stats.frames_dropped
+ return (
+ f"{processed}/{enqueue} frames | inference {processing_fps:.1f} fps | "
+ f"latency {latency_ms:.1f} ms (avg {avg_ms:.1f} ms) | "
+ f"queue {stats.queue_size} | dropped {dropped}"
+ )
+
+ def _update_metrics(self) -> None:
+ if hasattr(self, "camera_stats_label"):
+ if self.camera_controller.is_running():
+ fps = self._compute_fps(self._camera_frame_times)
+ if fps > 0:
+ self.camera_stats_label.setText(f"{fps:.1f} fps (last 5 s)")
+ else:
+ self.camera_stats_label.setText("Measuring…")
+ else:
+ self.camera_stats_label.setText("Camera idle")
+
+ if hasattr(self, "dlc_stats_label"):
+ if self._dlc_active and self._dlc_initialized:
+ stats = self.dlc_processor.get_stats()
+ summary = self._format_dlc_stats(stats)
+ self.dlc_stats_label.setText(summary)
+ else:
+ self.dlc_stats_label.setText("DLC processor idle")
+
+ if hasattr(self, "recording_stats_label"):
+ if self._video_recorder is not None:
+ stats = self._video_recorder.get_stats()
+ if stats is not None:
+ summary = self._format_recorder_stats(stats)
+ self._last_recorder_summary = summary
+ self.recording_stats_label.setText(summary)
+ elif not self._video_recorder.is_running:
+ self._last_recorder_summary = "Recorder idle"
+ self.recording_stats_label.setText(self._last_recorder_summary)
+ else:
+ self.recording_stats_label.setText(self._last_recorder_summary)
+
def _start_inference(self) -> None:
if self._dlc_active:
self.statusBar().showMessage("Pose inference already running", 3000)
@@ -734,17 +870,30 @@ def _start_inference(self) -> None:
self.dlc_processor.reset()
self._last_pose = None
self._dlc_active = True
- self.statusBar().showMessage("Starting pose inference…", 3000)
- self._update_inference_buttons()
+ self._dlc_initialized = False
+
+ # Update button to show initializing state
+ self.start_inference_button.setText("Initializing DLCLive!")
+ self.start_inference_button.setStyleSheet("background-color: #4A90E2; color: white;")
+ self.start_inference_button.setEnabled(False)
+ self.stop_inference_button.setEnabled(True)
+
+ self.statusBar().showMessage("Initializing DLCLive…", 3000)
self._update_camera_controls_enabled()
def _stop_inference(self, show_message: bool = True) -> None:
was_active = self._dlc_active
self._dlc_active = False
+ self._dlc_initialized = False
self.dlc_processor.reset()
self._last_pose = None
+
+ # Reset button appearance
+ self.start_inference_button.setText("Start pose inference")
+ self.start_inference_button.setStyleSheet("")
+
if self._current_frame is not None:
- self._update_video_display(self._current_frame)
+ self._display_frame(self._current_frame, force=True)
if was_active and show_message:
self.statusBar().showMessage("Pose inference stopped", 3000)
self._update_inference_buttons()
@@ -780,6 +929,7 @@ def _start_recording(self) -> None:
codec=recording.codec,
crf=recording.crf,
)
+ self._last_drop_warning = 0.0
try:
self._video_recorder.start()
except Exception as exc: # pragma: no cover - runtime error
@@ -788,16 +938,35 @@ def _start_recording(self) -> None:
return
self.start_record_button.setEnabled(False)
self.stop_record_button.setEnabled(True)
+ if hasattr(self, "recording_stats_label"):
+ self._last_recorder_summary = "Recorder running…"
+ self.recording_stats_label.setText(self._last_recorder_summary)
self.statusBar().showMessage(f"Recording to {output_path}", 5000)
self._update_camera_controls_enabled()
def _stop_recording(self) -> None:
if not self._video_recorder:
return
- self._video_recorder.stop()
+ recorder = self._video_recorder
+ recorder.stop()
+ stats = recorder.get_stats() if recorder is not None else None
self._video_recorder = None
self.start_record_button.setEnabled(True)
self.stop_record_button.setEnabled(False)
+ if hasattr(self, "recording_stats_label"):
+ if stats is not None:
+ summary = self._format_recorder_stats(stats)
+ else:
+ summary = "Recorder idle"
+ self._last_recorder_summary = summary
+ self.recording_stats_label.setText(summary)
+ else:
+ self._last_recorder_summary = (
+ self._format_recorder_stats(stats)
+ if stats is not None
+ else "Recorder idle"
+ )
+ self._last_drop_warning = 0.0
self.statusBar().showMessage("Recording stopped", 3000)
self._update_camera_controls_enabled()
@@ -808,26 +977,35 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
frame = self._apply_rotation(raw_frame)
frame = np.ascontiguousarray(frame)
self._current_frame = frame
+ self._track_camera_frame()
if self._active_camera_settings is not None:
height, width = frame.shape[:2]
self._active_camera_settings.width = int(width)
self._active_camera_settings.height = int(height)
if self._video_recorder and self._video_recorder.is_running:
try:
- self._video_recorder.write(frame)
+ success = self._video_recorder.write(frame)
+ if not success:
+ now = time.perf_counter()
+ if now - self._last_drop_warning > 1.0:
+ self.statusBar().showMessage(
+ "Recorder backlog full; dropping frames", 2000
+ )
+ self._last_drop_warning = now
except RuntimeError as exc:
self._show_error(str(exc))
self._stop_recording()
if self._dlc_active:
self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
- self._update_video_display(frame)
+ self._display_frame(frame)
def _on_pose_ready(self, result: PoseResult) -> None:
if not self._dlc_active:
return
self._last_pose = result
+ logging.info(f"Pose result: {result.pose}, Timestamp: {result.timestamp}")
if self._current_frame is not None:
- self._update_video_display(self._current_frame)
+ self._display_frame(self._current_frame, force=True)
def _on_dlc_error(self, message: str) -> None:
self._stop_inference(show_message=False)
@@ -858,7 +1036,7 @@ def _apply_rotation(self, frame: np.ndarray) -> np.ndarray:
def _on_show_predictions_changed(self, _state: int) -> None:
if self._current_frame is not None:
- self._update_video_display(self._current_frame)
+ self._display_frame(self._current_frame, force=True)
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
@@ -873,9 +1051,19 @@ def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
def _on_dlc_initialised(self, success: bool) -> None:
if success:
- self.statusBar().showMessage("DLCLive initialised", 3000)
+ self._dlc_initialized = True
+ # Update button to show running state
+ self.start_inference_button.setText("DLCLive running!")
+ self.start_inference_button.setStyleSheet("background-color: #4CAF50; color: white;")
+ self.statusBar().showMessage("DLCLive initialized successfully", 3000)
else:
- self.statusBar().showMessage("DLCLive initialisation failed", 3000)
+ self._dlc_initialized = False
+ # Reset button on failure
+ self.start_inference_button.setText("Start pose inference")
+ self.start_inference_button.setStyleSheet("")
+ self.statusBar().showMessage("DLCLive initialization failed", 5000)
+ # Stop inference since initialization failed
+ self._stop_inference(show_message=False)
# ------------------------------------------------------------------ helpers
def _show_error(self, message: str) -> None:
@@ -889,6 +1077,8 @@ def closeEvent(self, event: QCloseEvent) -> None: # pragma: no cover - GUI beha
if self._video_recorder and self._video_recorder.is_running:
self._video_recorder.stop()
self.dlc_processor.shutdown()
+ if hasattr(self, "_metrics_timer"):
+ self._metrics_timer.stop()
super().closeEvent(event)
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index ff52fdb..d729b02 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -1,6 +1,12 @@
"""Video recording support using the vidgear library."""
from __future__ import annotations
+import logging
+import queue
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
@@ -12,6 +18,26 @@
WriteGear = None # type: ignore[assignment]
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class RecorderStats:
+ """Snapshot of recorder throughput metrics."""
+
+ frames_enqueued: int
+ frames_written: int
+ dropped_frames: int
+ queue_size: int
+ average_latency: float
+ last_latency: float
+ write_fps: float
+ buffer_seconds: float
+
+
+_SENTINEL = object()
+
+
class VideoRecorder:
"""Thin wrapper around :class:`vidgear.gears.WriteGear`."""
@@ -22,17 +48,31 @@ def __init__(
frame_rate: Optional[float] = None,
codec: str = "libx264",
crf: int = 23,
+ buffer_size: int = 240,
):
self._output = Path(output)
- self._writer: Optional[WriteGear] = None
+ self._writer: Optional[Any] = None
self._frame_size = frame_size
self._frame_rate = frame_rate
self._codec = codec
self._crf = int(crf)
+ self._buffer_size = max(1, int(buffer_size))
+ self._queue: Optional[queue.Queue[Any]] = None
+ self._writer_thread: Optional[threading.Thread] = None
+ self._stop_event = threading.Event()
+ self._stats_lock = threading.Lock()
+ self._frames_enqueued = 0
+ self._frames_written = 0
+ self._dropped_frames = 0
+ self._total_latency = 0.0
+ self._last_latency = 0.0
+ self._written_times: deque[float] = deque(maxlen=600)
+ self._encode_error: Optional[Exception] = None
+ self._last_log_time = 0.0
@property
def is_running(self) -> bool:
- return self._writer is not None
+ return self._writer_thread is not None and self._writer_thread.is_alive()
def start(self) -> None:
if WriteGear is None:
@@ -54,6 +94,21 @@ def start(self) -> None:
self._output.parent.mkdir(parents=True, exist_ok=True)
self._writer = WriteGear(output=str(self._output), **writer_kwargs)
+ self._queue = queue.Queue(maxsize=self._buffer_size)
+ self._frames_enqueued = 0
+ self._frames_written = 0
+ self._dropped_frames = 0
+ self._total_latency = 0.0
+ self._last_latency = 0.0
+ self._written_times.clear()
+ self._encode_error = None
+ self._stop_event.clear()
+ self._writer_thread = threading.Thread(
+ target=self._writer_loop,
+ name="VideoRecorderWriter",
+ daemon=True,
+ )
+ self._writer_thread.start()
def configure_stream(
self, frame_size: Tuple[int, int], frame_rate: Optional[float]
@@ -61,9 +116,12 @@ def configure_stream(
self._frame_size = frame_size
self._frame_rate = frame_rate
- def write(self, frame: np.ndarray) -> None:
- if self._writer is None:
- return
+ def write(self, frame: np.ndarray) -> bool:
+ if not self.is_running or self._queue is None:
+ return False
+ error = self._current_error()
+ if error is not None:
+ raise RuntimeError(f"Video encoding failed: {error}") from error
if frame.dtype != np.uint8:
frame_float = frame.astype(np.float32, copy=False)
max_val = float(frame_float.max()) if frame_float.size else 0.0
@@ -75,19 +133,139 @@ def write(self, frame: np.ndarray) -> None:
frame = np.repeat(frame[:, :, None], 3, axis=2)
frame = np.ascontiguousarray(frame)
try:
- self._writer.write(frame)
- except OSError as exc:
- writer = self._writer
- self._writer = None
- if writer is not None:
- try:
- writer.close()
- except Exception:
- pass
- raise RuntimeError(f"Video encoding failed: {exc}") from exc
+ assert self._queue is not None
+ self._queue.put(frame, block=False)
+ except queue.Full:
+ with self._stats_lock:
+ self._dropped_frames += 1
+ queue_size = self._queue.qsize() if self._queue is not None else -1
+ logger.warning(
+ "Video recorder queue full; dropping frame. queue=%d buffer=%d",
+ queue_size,
+ self._buffer_size,
+ )
+ return False
+ with self._stats_lock:
+ self._frames_enqueued += 1
+ return True
def stop(self) -> None:
- if self._writer is None:
+ if self._writer is None and not self.is_running:
return
- self._writer.close()
+ self._stop_event.set()
+ if self._queue is not None:
+ try:
+ self._queue.put_nowait(_SENTINEL)
+ except queue.Full:
+ self._queue.put(_SENTINEL)
+ if self._writer_thread is not None:
+ self._writer_thread.join(timeout=5.0)
+ if self._writer_thread.is_alive():
+ logger.warning("Video recorder thread did not terminate cleanly")
+ if self._writer is not None:
+ try:
+ self._writer.close()
+ except Exception:
+ logger.exception("Failed to close WriteGear cleanly")
+ self._writer = None
+ self._writer_thread = None
+ self._queue = None
+
+ def get_stats(self) -> Optional[RecorderStats]:
+ if (
+ self._writer is None
+ and not self.is_running
+ and self._queue is None
+ and self._frames_enqueued == 0
+ and self._frames_written == 0
+ and self._dropped_frames == 0
+ ):
+ return None
+ queue_size = self._queue.qsize() if self._queue is not None else 0
+ with self._stats_lock:
+ frames_enqueued = self._frames_enqueued
+ frames_written = self._frames_written
+ dropped = self._dropped_frames
+ avg_latency = (
+ self._total_latency / self._frames_written
+ if self._frames_written
+ else 0.0
+ )
+ last_latency = self._last_latency
+ write_fps = self._compute_write_fps_locked()
+ buffer_seconds = queue_size * avg_latency if avg_latency > 0 else 0.0
+ return RecorderStats(
+ frames_enqueued=frames_enqueued,
+ frames_written=frames_written,
+ dropped_frames=dropped,
+ queue_size=queue_size,
+ average_latency=avg_latency,
+ last_latency=last_latency,
+ write_fps=write_fps,
+ buffer_seconds=buffer_seconds,
+ )
+
+ def _writer_loop(self) -> None:
+ assert self._queue is not None
+ while True:
+ try:
+ item = self._queue.get(timeout=0.1)
+ except queue.Empty:
+ if self._stop_event.is_set():
+ break
+ continue
+ if item is _SENTINEL:
+ self._queue.task_done()
+ break
+ frame = item
+ start = time.perf_counter()
+ try:
+ assert self._writer is not None
+ self._writer.write(frame)
+ except OSError as exc:
+ with self._stats_lock:
+ self._encode_error = exc
+ logger.exception("Video encoding failed while writing frame")
+ self._queue.task_done()
+ self._stop_event.set()
+ break
+ elapsed = time.perf_counter() - start
+ now = time.perf_counter()
+ with self._stats_lock:
+ self._frames_written += 1
+ self._total_latency += elapsed
+ self._last_latency = elapsed
+ self._written_times.append(now)
+ if now - self._last_log_time >= 1.0:
+ fps = self._compute_write_fps_locked()
+ queue_size = self._queue.qsize()
+ logger.info(
+ "Recorder throughput: %.2f fps, latency %.2f ms, queue=%d",
+ fps,
+ elapsed * 1000.0,
+ queue_size,
+ )
+ self._last_log_time = now
+ self._queue.task_done()
+ self._finalize_writer()
+
+ def _finalize_writer(self) -> None:
+ writer = self._writer
self._writer = None
+ if writer is not None:
+ try:
+ writer.close()
+ except Exception:
+ logger.exception("Failed to close WriteGear during finalisation")
+
+ def _compute_write_fps_locked(self) -> float:
+ if len(self._written_times) < 2:
+ return 0.0
+ duration = self._written_times[-1] - self._written_times[0]
+ if duration <= 0:
+ return 0.0
+ return (len(self._written_times) - 1) / duration
+
+ def _current_error(self) -> Optional[Exception]:
+ with self._stats_lock:
+ return self._encode_error
diff --git a/setup.py b/setup.py
index 163f8f0..a254101 100644
--- a/setup.py
+++ b/setup.py
@@ -15,7 +15,7 @@
long_description=long_description,
long_description_content_type="text/markdown",
url="https://github.com/DeepLabCut/DeepLabCut-live-GUI",
- python_requires=">=3.11",
+ python_requires=">=3.10",
install_requires=[
"deeplabcut-live",
"PyQt6",
@@ -25,7 +25,7 @@
],
extras_require={
"basler": ["pypylon"],
- "gentl": ["pygobject"],
+ "gentl": ["harvesters"],
},
packages=setuptools.find_packages(),
include_package_data=True,
From 6402566d540354cfb6cc894681a9ffee7af9db1a Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 23 Oct 2025 16:00:39 +0200
Subject: [PATCH 08/26] Added processors
---
dlclivegui/processors/PLUGIN_SYSTEM.md | 191 +++++++
dlclivegui/processors/dlc_processor_socket.py | 509 ++++++++++++++++++
dlclivegui/processors/processor_utils.py | 83 +++
3 files changed, 783 insertions(+)
create mode 100644 dlclivegui/processors/PLUGIN_SYSTEM.md
create mode 100644 dlclivegui/processors/dlc_processor_socket.py
create mode 100644 dlclivegui/processors/processor_utils.py
diff --git a/dlclivegui/processors/PLUGIN_SYSTEM.md b/dlclivegui/processors/PLUGIN_SYSTEM.md
new file mode 100644
index 0000000..b02402d
--- /dev/null
+++ b/dlclivegui/processors/PLUGIN_SYSTEM.md
@@ -0,0 +1,191 @@
+# DLC Processor Plugin System
+
+This folder contains a plugin-style architecture for DLC processors that allows GUI tools to discover and instantiate processors dynamically.
+
+## Architecture
+
+### 1. Processor Registry
+
+Each processor file should define a `PROCESSOR_REGISTRY` dictionary and helper functions:
+
+```python
+# Registry for GUI discovery
+PROCESSOR_REGISTRY = {}
+
+# At end of file, register your processors
+PROCESSOR_REGISTRY["MyProcessor_socket"] = MyProcessor_socket
+```
+
+### 2. Processor Metadata
+
+Each processor class should define metadata attributes for GUI discovery:
+
+```python
+class MyProcessor_socket(BaseProcessor_socket):
+ # Metadata for GUI discovery
+ PROCESSOR_NAME = "Mouse Pose Processor" # Human-readable name
+ PROCESSOR_DESCRIPTION = "Calculates mouse center, heading, and head angle"
+ PROCESSOR_PARAMS = {
+ "bind": {
+ "type": "tuple",
+ "default": ("0.0.0.0", 6000),
+ "description": "Server address (host, port)"
+ },
+ "use_filter": {
+ "type": "bool",
+ "default": False,
+ "description": "Apply One-Euro filter"
+ },
+ # ... more parameters
+ }
+```
+
+### 3. Discovery Functions
+
+Two helper functions enable GUI discovery:
+
+```python
+def get_available_processors():
+ """Returns dict of available processors with metadata."""
+
+def instantiate_processor(class_name, **kwargs):
+ """Instantiates a processor by name with given parameters."""
+```
+
+## GUI Integration
+
+### Simple Usage
+
+```python
+from dlc_processor_socket import get_available_processors, instantiate_processor
+
+# 1. Get available processors
+processors = get_available_processors()
+
+# 2. Display to user (e.g., in dropdown)
+for class_name, info in processors.items():
+ print(f"{info['name']} - {info['description']}")
+
+# 3. User selects "MyProcessor_socket"
+selected_class = "MyProcessor_socket"
+
+# 4. Show parameter form based on info['params']
+processor_info = processors[selected_class]
+for param_name, param_info in processor_info['params'].items():
+ # Create input widget for param_type and default value
+ pass
+
+# 5. Instantiate with user's values
+processor = instantiate_processor(
+ selected_class,
+ bind=("127.0.0.1", 7000),
+ use_filter=True
+)
+```
+
+### Scanning Multiple Files
+
+To scan a folder for processor files:
+
+```python
+import importlib.util
+from pathlib import Path
+
+def load_processors_from_file(file_path):
+ """Load processors from a single file."""
+ spec = importlib.util.spec_from_file_location("processors", file_path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ if hasattr(module, 'get_available_processors'):
+ return module.get_available_processors()
+ return {}
+
+# Scan folder
+for py_file in Path("dlc_processors").glob("*.py"):
+ processors = load_processors_from_file(py_file)
+ # Display processors to user
+```
+
+## Examples
+
+### 1. Command-line Example
+
+```bash
+python example_gui_usage.py
+```
+
+This demonstrates:
+- Loading processors
+- Displaying metadata
+- Instantiating with default/custom parameters
+- Simulated GUI workflow
+
+### 2. tkinter GUI
+
+```bash
+python processor_gui_simple.py
+```
+
+This provides a full GUI with:
+- Dropdown to select processor
+- Auto-generated parameter form
+- Create/Stop buttons
+- Status display
+
+## Adding New Processors
+
+To make a new processor discoverable:
+
+1. **Define metadata attributes:**
+```python
+class MyNewProcessor(BaseProcessor_socket):
+ PROCESSOR_NAME = "My New Processor"
+ PROCESSOR_DESCRIPTION = "Does something cool"
+ PROCESSOR_PARAMS = {
+ "my_param": {
+ "type": "bool",
+ "default": True,
+ "description": "Enable cool feature"
+ }
+ }
+```
+
+2. **Register in PROCESSOR_REGISTRY:**
+```python
+PROCESSOR_REGISTRY["MyNewProcessor"] = MyNewProcessor
+```
+
+3. **Done!** GUI will automatically discover it.
+
+## Parameter Types
+
+Supported parameter types in `PROCESSOR_PARAMS`:
+
+- `"bool"` - Boolean checkbox
+- `"int"` - Integer input
+- `"float"` - Float input
+- `"str"` - String input
+- `"bytes"` - String that gets encoded to bytes
+- `"tuple"` - Tuple (e.g., `(host, port)`)
+- `"dict"` - Dictionary (e.g., filter parameters)
+- `"list"` - List
+
+## Benefits
+
+1. **No hardcoding** - GUI doesn't need to know about specific processors
+2. **Easy extension** - Add new processors without modifying GUI code
+3. **Self-documenting** - Parameters include descriptions
+4. **Type-safe** - Parameter metadata includes type information
+5. **Modular** - Each processor file can be independent
+
+## File Structure
+
+```
+dlc_processors/
+├── dlc_processor_socket.py # Base + MyProcessor with registry
+├── my_custom_processor.py # Your custom processor (with registry)
+├── example_gui_usage.py # Command-line example
+├── processor_gui_simple.py # tkinter GUI example
+└── PLUGIN_SYSTEM.md # This file
+```
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
new file mode 100644
index 0000000..bd183af
--- /dev/null
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -0,0 +1,509 @@
+import logging
+import pickle
+import time
+from collections import deque
+from math import acos, atan2, copysign, degrees, pi, sqrt
+from multiprocessing.connection import Listener
+from threading import Event, Thread
+
+import numpy as np
+from dlclive import Processor
+
+LOG = logging.getLogger("dlc_processor_socket")
+LOG.setLevel(logging.INFO)
+_handler = logging.StreamHandler()
+_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
+LOG.addHandler(_handler)
+
+
+# Registry for GUI discovery
+PROCESSOR_REGISTRY = {}
+
+
+class OneEuroFilter:
+ def __init__(self, t0, x0, dx0=None, min_cutoff=1.0, beta=0.0, d_cutoff=1.0):
+ self.min_cutoff = min_cutoff
+ self.beta = beta
+ self.d_cutoff = d_cutoff
+ self.x_prev = x0
+ if dx0 is None:
+ dx0 = np.zeros_like(x0)
+ self.dx_prev = dx0
+ self.t_prev = t0
+
+ @staticmethod
+ def smoothing_factor(t_e, cutoff):
+ r = 2 * pi * cutoff * t_e
+ return r / (r + 1)
+
+ @staticmethod
+ def exponential_smoothing(alpha, x, x_prev):
+ return alpha * x + (1 - alpha) * x_prev
+
+ def __call__(self, t, x):
+ t_e = t - self.t_prev
+
+ a_d = self.smoothing_factor(t_e, self.d_cutoff)
+ dx = (x - self.x_prev) / t_e
+ dx_hat = self.exponential_smoothing(a_d, dx, self.dx_prev)
+
+ cutoff = self.min_cutoff + self.beta * abs(dx_hat)
+ a = self.smoothing_factor(t_e, cutoff)
+ x_hat = self.exponential_smoothing(a, x, self.x_prev)
+
+ self.x_prev = x_hat
+ self.dx_prev = dx_hat
+ self.t_prev = t
+
+ return x_hat
+
+
+class BaseProcessor_socket(Processor):
+ """
+ Base DLC Processor with multi-client broadcasting support.
+
+ Handles network connections, timing, and data logging.
+ Subclasses should implement custom pose processing logic.
+ """
+
+ # Metadata for GUI discovery
+ PROCESSOR_NAME = "Base Socket Processor"
+ PROCESSOR_DESCRIPTION = "Base class for socket-based processors with multi-client support"
+ PROCESSOR_PARAMS = {
+ "bind": {
+ "type": "tuple",
+ "default": ("0.0.0.0", 6000),
+ "description": "Server address (host, port)"
+ },
+ "authkey": {
+ "type": "bytes",
+ "default": b"secret password",
+ "description": "Authentication key for clients"
+ },
+ "use_perf_counter": {
+ "type": "bool",
+ "default": False,
+ "description": "Use time.perf_counter() instead of time.time()"
+ },
+ "save_original": {
+ "type": "bool",
+ "default": False,
+ "description": "Save raw pose arrays for analysis"
+ }
+ }
+
+ def __init__(
+ self,
+ bind=("0.0.0.0", 6000),
+ authkey=b"secret password",
+ use_perf_counter=False,
+ save_original=False,
+ ):
+ """
+ Initialize base processor with socket server.
+
+ Args:
+ bind: (host, port) tuple for server binding
+ authkey: Authentication key for client connections
+ use_perf_counter: If True, use time.perf_counter() instead of time.time()
+ save_original: If True, save raw pose arrays for analysis
+ """
+ super().__init__()
+
+ # Network setup
+ self.address = bind
+ self.authkey = authkey
+ self.listener = Listener(bind, authkey=authkey)
+ self._stop = Event()
+ self.conns = set()
+
+ # Start accept loop in background
+ Thread(target=self._accept_loop, name="DLCAccept", daemon=True).start()
+
+ # Timing function
+ self.timing_func = time.perf_counter if use_perf_counter else time.time
+ self.start_time = self.timing_func()
+
+ # Data storage
+ self.time_stamp = deque()
+ self.step = deque()
+ self.frame_time = deque()
+ self.pose_time = deque()
+ self.original_pose = deque()
+
+ # State
+ self.curr_step = 0
+ self.save_original = save_original
+
+ def _accept_loop(self):
+ """Background thread to accept new client connections."""
+ LOG.info(f"DLC Processor listening on {self.address[0]}:{self.address[1]}")
+ while not self._stop.is_set():
+ try:
+ c = self.listener.accept()
+ LOG.info(f"Client connected from {self.listener.last_accepted}")
+ self.conns.add(c)
+ # Start RX loop for this connection (in case clients send data)
+ Thread(target=self._rx_loop, args=(c,), name="DLCRX", daemon=True).start()
+ except (OSError, EOFError):
+ break
+
+ def _rx_loop(self, c):
+ """Background thread to handle receive from a client (detects disconnects)."""
+ while not self._stop.is_set():
+ try:
+ if c.poll(0.05):
+ msg = c.recv()
+ # Optional: handle client messages here
+ except (EOFError, OSError, BrokenPipeError):
+ break
+ try:
+ c.close()
+ except Exception:
+ pass
+ self.conns.discard(c)
+ LOG.info("Client disconnected")
+
+ def broadcast(self, payload):
+ """Send payload to all connected clients."""
+ dead = []
+ for c in list(self.conns):
+ try:
+ c.send(payload)
+ except (EOFError, OSError, BrokenPipeError):
+ dead.append(c)
+ for c in dead:
+ try:
+ c.close()
+ except Exception:
+ pass
+ self.conns.discard(c)
+
+ def process(self, pose, **kwargs):
+ """
+ Process pose and broadcast to clients.
+
+ This base implementation just saves original pose and broadcasts it.
+ Subclasses should override to add custom processing.
+
+ Args:
+ pose: DLC pose array (N_keypoints x 3) with [x, y, confidence]
+ **kwargs: Additional metadata (frame_time, pose_time, etc.)
+
+ Returns:
+ pose: Unmodified pose array
+ """
+ curr_time = self.timing_func()
+
+ # Save original pose if requested
+ if self.save_original:
+ self.original_pose.append(pose.copy())
+
+ # Update step counter
+ self.curr_step = self.curr_step + 1
+
+ # Store metadata
+ self.time_stamp.append(curr_time)
+ self.step.append(self.curr_step)
+ self.frame_time.append(kwargs.get("frame_time", -1))
+ if "pose_time" in kwargs:
+ self.pose_time.append(kwargs["pose_time"])
+
+ # Broadcast raw pose to all connected clients
+ payload = [curr_time, pose]
+ self.broadcast(payload)
+
+ return pose
+
+ def stop(self):
+ """Stop the processor and close all connections."""
+ self._stop.set()
+ try:
+ self.listener.close()
+ except Exception:
+ pass
+ for c in list(self.conns):
+ try:
+ c.close()
+ except Exception:
+ pass
+ self.conns.discard(c)
+ LOG.info("Processor stopped, all connections closed")
+
+ def save(self, file=None):
+ """Save logged data to file."""
+ save_code = 0
+ if file:
+ LOG.info(f"Saving data to {file}")
+ try:
+ save_dict = self.get_data()
+ pickle.dump(save_dict, open(file, "wb"))
+ save_code = 1
+ except Exception as e:
+ LOG.error(f"Save failed: {e}")
+ save_code = -1
+ return save_code
+
+ def get_data(self):
+ """Get logged data as dictionary."""
+ save_dict = dict()
+ if self.save_original:
+ save_dict["original_pose"] = np.array(self.original_pose)
+ save_dict["start_time"] = self.start_time
+ save_dict["time_stamp"] = np.array(self.time_stamp)
+ save_dict["step"] = np.array(self.step)
+ save_dict["frame_time"] = np.array(self.frame_time)
+ save_dict["pose_time"] = np.array(self.pose_time) if self.pose_time else None
+ save_dict["use_perf_counter"] = self.timing_func == time.perf_counter
+ return save_dict
+
+
+class MyProcessor_socket(BaseProcessor_socket):
+ """
+ DLC Processor with pose calculations (center, heading, head angle) and optional filtering.
+
+ Calculates:
+ - center: Weighted average of head keypoints
+ - heading: Body orientation (degrees)
+ - head_angle: Head rotation relative to body (radians)
+
+ Broadcasts: [timestamp, center_x, center_y, heading, head_angle]
+ """
+
+ # Metadata for GUI discovery
+ PROCESSOR_NAME = "Mouse Pose Processor"
+ PROCESSOR_DESCRIPTION = "Calculates mouse center, heading, and head angle with optional One-Euro filtering"
+ PROCESSOR_PARAMS = {
+ "bind": {
+ "type": "tuple",
+ "default": ("0.0.0.0", 6000),
+ "description": "Server address (host, port)"
+ },
+ "authkey": {
+ "type": "bytes",
+ "default": b"secret password",
+ "description": "Authentication key for clients"
+ },
+ "use_perf_counter": {
+ "type": "bool",
+ "default": False,
+ "description": "Use time.perf_counter() instead of time.time()"
+ },
+ "use_filter": {
+ "type": "bool",
+ "default": False,
+ "description": "Apply One-Euro filter to calculated values"
+ },
+ "filter_kwargs": {
+ "type": "dict",
+ "default": {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0},
+ "description": "One-Euro filter parameters (min_cutoff, beta, d_cutoff)"
+ },
+ "save_original": {
+ "type": "bool",
+ "default": False,
+ "description": "Save raw pose arrays for analysis"
+ }
+ }
+
+ def __init__(
+ self,
+ bind=("0.0.0.0", 6000),
+ authkey=b"secret password",
+ use_perf_counter=False,
+ use_filter=False,
+ filter_kwargs=None,
+ save_original=False,
+ ):
+ """
+ DLC Processor with multi-client broadcasting support.
+
+ Args:
+ bind: (host, port) tuple for server binding
+ authkey: Authentication key for client connections
+ use_perf_counter: If True, use time.perf_counter() instead of time.time()
+ use_filter: If True, apply One-Euro filter to pose data
+ filter_kwargs: Dict with OneEuroFilter parameters (min_cutoff, beta, d_cutoff)
+ save_original: If True, save raw pose arrays
+ """
+ super().__init__(
+ bind=bind,
+ authkey=authkey,
+ use_perf_counter=use_perf_counter,
+ save_original=save_original,
+ )
+
+ # Additional data storage for processed values
+ self.center_x = deque()
+ self.center_y = deque()
+ self.heading_direction = deque()
+ self.head_angle = deque()
+
+ # Filtering
+ self.use_filter = use_filter
+ self.filter_kwargs = filter_kwargs or {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0}
+ self.filters = None # Will be initialized on first pose
+
+ def _initialize_filters(self, vals):
+ """Initialize One-Euro filters for each output variable."""
+ t0 = self.timing_func()
+ self.filters = {
+ "center_x": OneEuroFilter(t0, vals[0], **self.filter_kwargs),
+ "center_y": OneEuroFilter(t0, vals[1], **self.filter_kwargs),
+ "heading": OneEuroFilter(t0, vals[2], **self.filter_kwargs),
+ "head_angle": OneEuroFilter(t0, vals[3], **self.filter_kwargs),
+ }
+ LOG.debug(f"Initialized One-Euro filters with parameters: {self.filter_kwargs}")
+
+ def process(self, pose, **kwargs):
+ """
+ Process pose: calculate center/heading/head_angle, optionally filter, and broadcast.
+
+ Args:
+ pose: DLC pose array (N_keypoints x 3) with [x, y, confidence]
+ **kwargs: Additional metadata (frame_time, pose_time, etc.)
+
+ Returns:
+ pose: Unmodified pose array
+ """
+ # Save original pose if requested (from base class)
+ if self.save_original:
+ self.original_pose.append(pose.copy())
+
+ # Extract keypoints and confidence
+ xy = pose[:, :2]
+ conf = pose[:, 2]
+
+ # Calculate weighted center from head keypoints
+ head_xy = xy[[0, 1, 2, 3, 4, 5, 6, 26], :]
+ head_conf = conf[[0, 1, 2, 3, 4, 5, 6, 26]]
+ center = np.average(head_xy, axis=0, weights=head_conf)
+
+ # Calculate body axis (tail_base -> neck)
+ body_axis = xy[7] - xy[13]
+ body_axis /= sqrt(np.sum(body_axis**2))
+
+ # Calculate head axis (neck -> nose)
+ head_axis = xy[0] - xy[7]
+ head_axis /= sqrt(np.sum(head_axis**2))
+
+ # Calculate head angle relative to body
+ cross = body_axis[0] * head_axis[1] - head_axis[0] * body_axis[1]
+ sign = copysign(1, cross) # Positive when looking left
+ try:
+ head_angle = acos(body_axis @ head_axis) * sign
+ except ValueError:
+ head_angle = 0
+
+ # Calculate heading (body orientation)
+ heading = atan2(body_axis[1], body_axis[0])
+ heading = degrees(heading)
+
+ # Raw values (heading unwrapped for filtering)
+ vals = [center[0], center[1], heading, head_angle]
+
+ # Apply filtering if enabled
+ curr_time = self.timing_func()
+ if self.use_filter:
+ if self.filters is None:
+ self._initialize_filters(vals)
+
+ # Filter each value (heading is filtered in unwrapped space)
+ filtered_vals = [
+ self.filters["center_x"](curr_time, vals[0]),
+ self.filters["center_y"](curr_time, vals[1]),
+ self.filters["heading"](curr_time, vals[2]),
+ self.filters["head_angle"](curr_time, vals[3]),
+ ]
+ vals = filtered_vals
+
+ # Wrap heading to [0, 360) after filtering
+ vals[2] = vals[2] % 360
+
+ # Update step counter
+ self.curr_step = self.curr_step + 1
+
+ # Store processed data
+ self.center_x.append(vals[0])
+ self.center_y.append(vals[1])
+ self.heading_direction.append(vals[2])
+ self.head_angle.append(vals[3])
+ self.time_stamp.append(curr_time)
+ self.step.append(self.curr_step)
+ self.frame_time.append(kwargs.get("frame_time", -1))
+ if "pose_time" in kwargs:
+ self.pose_time.append(kwargs["pose_time"])
+
+ # Broadcast processed values to all connected clients
+ payload = [curr_time, vals[0], vals[1], vals[2], vals[3]]
+ self.broadcast(payload)
+
+ return pose
+
+ def get_data(self):
+ """Get logged data including base class data and processed values."""
+ # Get base class data
+ save_dict = super().get_data()
+
+ # Add processed values
+ save_dict["x_pos"] = np.array(self.center_x)
+ save_dict["y_pos"] = np.array(self.center_y)
+ save_dict["heading_direction"] = np.array(self.heading_direction)
+ save_dict["head_angle"] = np.array(self.head_angle)
+ save_dict["use_filter"] = self.use_filter
+ save_dict["filter_kwargs"] = self.filter_kwargs
+
+ return save_dict
+
+
+# Register processors for GUI discovery
+PROCESSOR_REGISTRY["BaseProcessor_socket"] = BaseProcessor_socket
+PROCESSOR_REGISTRY["MyProcessor_socket"] = MyProcessor_socket
+
+
+def get_available_processors():
+ """
+ Get list of available processor classes.
+
+ Returns:
+ dict: Dictionary mapping class names to processor info:
+ {
+ "ClassName": {
+ "class": ProcessorClass,
+ "name": "Display Name",
+ "description": "Description text",
+ "params": {...}
+ }
+ }
+ """
+ processors = {}
+ for class_name, processor_class in PROCESSOR_REGISTRY.items():
+ processors[class_name] = {
+ "class": processor_class,
+ "name": getattr(processor_class, "PROCESSOR_NAME", class_name),
+ "description": getattr(processor_class, "PROCESSOR_DESCRIPTION", ""),
+ "params": getattr(processor_class, "PROCESSOR_PARAMS", {})
+ }
+ return processors
+
+
+def instantiate_processor(class_name, **kwargs):
+ """
+ Instantiate a processor by class name with given parameters.
+
+ Args:
+ class_name: Name of the processor class (e.g., "MyProcessor_socket")
+ **kwargs: Parameters to pass to the processor constructor
+
+ Returns:
+ Processor instance
+
+ Raises:
+ ValueError: If class_name is not in registry
+ """
+ if class_name not in PROCESSOR_REGISTRY:
+ available = ", ".join(PROCESSOR_REGISTRY.keys())
+ raise ValueError(f"Unknown processor '{class_name}'. Available: {available}")
+
+ processor_class = PROCESSOR_REGISTRY[class_name]
+ return processor_class(**kwargs)
diff --git a/dlclivegui/processors/processor_utils.py b/dlclivegui/processors/processor_utils.py
new file mode 100644
index 0000000..728ff87
--- /dev/null
+++ b/dlclivegui/processors/processor_utils.py
@@ -0,0 +1,83 @@
+
+import importlib.util
+import inspect
+from pathlib import Path
+
+
+def load_processors_from_file(file_path):
+ """
+ Load all processor classes from a Python file.
+
+ Args:
+ file_path: Path to Python file containing processors
+
+ Returns:
+ dict: Dictionary of available processors
+ """
+ # Load module from file
+ spec = importlib.util.spec_from_file_location("processors", file_path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+
+ # Check if module has get_available_processors function
+ if hasattr(module, 'get_available_processors'):
+ return module.get_available_processors()
+
+ # Fallback: scan for Processor subclasses
+ from dlclive import Processor
+ processors = {}
+ for name, obj in inspect.getmembers(module, inspect.isclass):
+ if issubclass(obj, Processor) and obj != Processor:
+ processors[name] = {
+ "class": obj,
+ "name": getattr(obj, "PROCESSOR_NAME", name),
+ "description": getattr(obj, "PROCESSOR_DESCRIPTION", ""),
+ "params": getattr(obj, "PROCESSOR_PARAMS", {})
+ }
+ return processors
+
+
+def scan_processor_folder(folder_path):
+ """
+ Scan a folder for all Python files with processor definitions.
+
+ Args:
+ folder_path: Path to folder containing processor files
+
+ Returns:
+ dict: Dictionary mapping file names to their processors
+ """
+ all_processors = {}
+ folder = Path(folder_path)
+
+ for py_file in folder.glob("*.py"):
+ if py_file.name.startswith("_"):
+ continue
+ elif py_file.name == "processor_utils.py":
+ continue
+ try:
+ processors = load_processors_from_file(py_file)
+ if processors:
+ all_processors[py_file.name] = processors
+ except Exception as e:
+ print(f"Error loading {py_file}: {e}")
+
+ return all_processors
+
+
+def display_processor_info(processors):
+ """Display processor information in a user-friendly format."""
+ print("\n" + "="*70)
+ print("AVAILABLE PROCESSORS")
+ print("="*70)
+
+ for idx, (class_name, info) in enumerate(processors.items(), 1):
+ print(f"\n[{idx}] {info['name']}")
+ print(f" Class: {class_name}")
+ print(f" Description: {info['description']}")
+ print(f" Parameters:")
+ for param_name, param_info in info['params'].items():
+ print(f" - {param_name} ({param_info['type']})")
+ print(f" Default: {param_info['default']}")
+ print(f" {param_info['description']}")
+
From c81e8972d3285241a36bb314b70191123c058c3b Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 23 Oct 2025 16:24:42 +0200
Subject: [PATCH 09/26] update processors
---
dlclivegui/processors/GUI_INTEGRATION.md | 167 +++++++++++++++++++++++
dlclivegui/processors/processor_utils.py | 56 +++++++-
2 files changed, 217 insertions(+), 6 deletions(-)
create mode 100644 dlclivegui/processors/GUI_INTEGRATION.md
diff --git a/dlclivegui/processors/GUI_INTEGRATION.md b/dlclivegui/processors/GUI_INTEGRATION.md
new file mode 100644
index 0000000..446232e
--- /dev/null
+++ b/dlclivegui/processors/GUI_INTEGRATION.md
@@ -0,0 +1,167 @@
+# GUI Integration Guide
+
+## Quick Answer
+
+Here's how to use `scan_processor_folder` in your GUI:
+
+```python
+from example_gui_usage import scan_processor_folder, instantiate_from_scan
+
+# 1. Scan folder
+all_processors = scan_processor_folder("./processors")
+
+# 2. Populate dropdown with keys (for backend) and display names (for user)
+for key, info in all_processors.items():
+ # key = "file.py::ClassName" (use this for instantiation)
+ # display_name = "Human Name (file.py)" (show this to user)
+ display_name = f"{info['name']} ({info['file']})"
+ dropdown.add_item(key, display_name)
+
+# 3. When user selects, get the key from dropdown
+selected_key = dropdown.get_selected_value() # e.g., "dlc_processor_socket.py::MyProcessor_socket"
+
+# 4. Get processor info
+processor_info = all_processors[selected_key]
+
+# 5. Build parameter form from processor_info['params']
+for param_name, param_info in processor_info['params'].items():
+ add_input_field(param_name, param_info['type'], param_info['default'])
+
+# 6. When user clicks Create, instantiate using the key
+user_params = get_form_values()
+processor = instantiate_from_scan(all_processors, selected_key, **user_params)
+```
+
+## The Key Insight
+
+**The key returned by `scan_processor_folder` is what you use to instantiate!**
+
+```python
+# OLD problem: "I have a name, how do I load it?"
+# NEW solution: Use the key directly
+
+all_processors = scan_processor_folder(folder)
+# Returns: {"file.py::ClassName": {processor_info}, ...}
+
+# The KEY "file.py::ClassName" uniquely identifies the processor
+# Pass this key to instantiate_from_scan()
+
+processor = instantiate_from_scan(all_processors, "file.py::ClassName", **params)
+```
+
+## What's in the returned dict?
+
+```python
+all_processors = {
+ "dlc_processor_socket.py::MyProcessor_socket": {
+ "class": , # The actual class
+ "name": "Mouse Pose Processor", # Human-readable name
+ "description": "Calculates mouse...", # Description
+ "params": { # All parameters
+ "bind": {
+ "type": "tuple",
+ "default": ("0.0.0.0", 6000),
+ "description": "Server address"
+ },
+ # ... more parameters
+ },
+ "file": "dlc_processor_socket.py", # Source file
+ "class_name": "MyProcessor_socket", # Class name
+ "file_path": "/full/path/to/file.py" # Full path
+ }
+}
+```
+
+## GUI Workflow
+
+### Step 1: Scan Folder
+```python
+all_processors = scan_processor_folder("./processors")
+```
+
+### Step 2: Populate Dropdown
+```python
+# Store keys in order (for mapping dropdown index -> key)
+self.processor_keys = list(all_processors.keys())
+
+# Create display names for dropdown
+display_names = [
+ f"{info['name']} ({info['file']})"
+ for info in all_processors.values()
+]
+dropdown.set_items(display_names)
+```
+
+### Step 3: User Selects Processor
+```python
+def on_processor_selected(dropdown_index):
+ # Get the key
+ key = self.processor_keys[dropdown_index]
+
+ # Get processor info
+ info = all_processors[key]
+
+ # Show description
+ description_label.text = info['description']
+
+ # Build parameter form
+ for param_name, param_info in info['params'].items():
+ add_parameter_field(
+ name=param_name,
+ type=param_info['type'],
+ default=param_info['default'],
+ help_text=param_info['description']
+ )
+```
+
+### Step 4: User Clicks Create
+```python
+def on_create_clicked():
+ # Get selected key
+ key = self.processor_keys[dropdown.current_index]
+
+ # Get user's parameter values
+ user_params = parameter_form.get_values()
+
+ # Instantiate using the key!
+ self.processor = instantiate_from_scan(
+ all_processors,
+ key,
+ **user_params
+ )
+
+ print(f"Created: {self.processor.__class__.__name__}")
+```
+
+## Why This Works
+
+1. **Unique Keys**: `"file.py::ClassName"` format ensures uniqueness even if multiple files have same class name
+
+2. **All Info Included**: Each dict entry has everything needed (class, metadata, parameters)
+
+3. **Simple Lookup**: Just use the key to get processor info or instantiate
+
+4. **No Manual Imports**: `scan_processor_folder` handles all module loading
+
+5. **Type Safety**: Parameter metadata includes types for validation
+
+## Complete Example
+
+See `processor_gui.py` for a full working tkinter GUI that demonstrates:
+- Folder scanning
+- Processor selection
+- Parameter form generation
+- Instantiation
+
+Run it with:
+```bash
+python processor_gui.py
+```
+
+## Files
+
+- `dlc_processor_socket.py` - Processors with metadata and registry
+- `example_gui_usage.py` - Scanning and instantiation functions + examples
+- `processor_gui.py` - Full tkinter GUI
+- `GUI_USAGE_GUIDE.py` - Pseudocode and examples
+- `README.md` - Documentation on the plugin system
diff --git a/dlclivegui/processors/processor_utils.py b/dlclivegui/processors/processor_utils.py
index 728ff87..dcacaa4 100644
--- a/dlclivegui/processors/processor_utils.py
+++ b/dlclivegui/processors/processor_utils.py
@@ -11,7 +11,7 @@ def load_processors_from_file(file_path):
Args:
file_path: Path to Python file containing processors
- Returns:
+ Returns:/home/as153/work_geneva/mice_ar_tasks/mouse_ar/ctrl/dlc_processors/GUI_INTEGRATION.md
dict: Dictionary of available processors
"""
# Load module from file
@@ -45,7 +45,17 @@ def scan_processor_folder(folder_path):
folder_path: Path to folder containing processor files
Returns:
- dict: Dictionary mapping file names to their processors
+ dict: Dictionary mapping unique processor keys to processor info:
+ {
+ "file_name.py::ClassName": {
+ "class": ProcessorClass,
+ "name": "Display Name",
+ "description": "...",
+ "params": {...},
+ "file": "file_name.py",
+ "class_name": "ClassName"
+ }
+ }
"""
all_processors = {}
folder = Path(folder_path)
@@ -53,18 +63,52 @@ def scan_processor_folder(folder_path):
for py_file in folder.glob("*.py"):
if py_file.name.startswith("_"):
continue
- elif py_file.name == "processor_utils.py":
- continue
+
try:
processors = load_processors_from_file(py_file)
- if processors:
- all_processors[py_file.name] = processors
+ for class_name, processor_info in processors.items():
+ # Create unique key: file::class
+ key = f"{py_file.name}::{class_name}"
+ # Add file and class name to info
+ processor_info["file"] = py_file.name
+ processor_info["class_name"] = class_name
+ processor_info["file_path"] = str(py_file)
+ all_processors[key] = processor_info
except Exception as e:
print(f"Error loading {py_file}: {e}")
return all_processors
+def instantiate_from_scan(processors_dict, processor_key, **kwargs):
+ """
+ Instantiate a processor from scan_processor_folder results.
+
+ Args:
+ processors_dict: Dict returned by scan_processor_folder
+ processor_key: Key like "file.py::ClassName"
+ **kwargs: Parameters for processor constructor
+
+ Returns:
+ Processor instance
+
+ Example:
+ processors = scan_processor_folder("./dlc_processors")
+ processor = instantiate_from_scan(
+ processors,
+ "dlc_processor_socket.py::MyProcessor_socket",
+ use_filter=True
+ )
+ """
+ if processor_key not in processors_dict:
+ available = ", ".join(processors_dict.keys())
+ raise ValueError(f"Unknown processor '{processor_key}'. Available: {available}")
+
+ processor_info = processors_dict[processor_key]
+ processor_class = processor_info["class"]
+ return processor_class(**kwargs)
+
+
def display_processor_info(processors):
"""Display processor information in a user-friendly format."""
print("\n" + "="*70)
From c7ee2f11ef26d984612c5f564bf0a207d28b35db Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 23 Oct 2025 17:54:08 +0200
Subject: [PATCH 10/26] fixing sizes
---
dlclivegui/camera_controller.py | 192 +++++++++++++++++++++++++--
dlclivegui/cameras/factory.py | 2 -
dlclivegui/cameras/gentl_backend.py | 21 +--
dlclivegui/cameras/opencv_backend.py | 52 +++++---
dlclivegui/config.py | 20 ++-
dlclivegui/dlc_processor.py | 2 +-
dlclivegui/gui.py | 138 ++++++++++++++-----
dlclivegui/video_recorder.py | 86 ++++++++++--
8 files changed, 415 insertions(+), 98 deletions(-)
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 821a252..5618163 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -1,6 +1,8 @@
"""Camera management for the DLC Live GUI."""
from __future__ import annotations
+import logging
+import time
from dataclasses import dataclass
from threading import Event
from typing import Optional
@@ -12,6 +14,8 @@
from dlclivegui.cameras.base import CameraBackend
from dlclivegui.config import CameraSettings
+LOGGER = logging.getLogger(__name__)
+
@dataclass
class FrameData:
@@ -27,6 +31,7 @@ class CameraWorker(QObject):
frame_captured = pyqtSignal(object)
started = pyqtSignal(object)
error_occurred = pyqtSignal(str)
+ warning_occurred = pyqtSignal(str)
finished = pyqtSignal()
def __init__(self, settings: CameraSettings):
@@ -34,42 +39,199 @@ def __init__(self, settings: CameraSettings):
self._settings = settings
self._stop_event = Event()
self._backend: Optional[CameraBackend] = None
+
+ # Error recovery settings
+ self._max_consecutive_errors = 5
+ self._max_reconnect_attempts = 3
+ self._retry_delay = 0.1 # seconds
+ self._reconnect_delay = 1.0 # seconds
+
+ # Frame validation
+ self._expected_frame_size: Optional[tuple[int, int]] = None # (height, width)
@pyqtSlot()
def run(self) -> None:
self._stop_event.clear()
- try:
- self._backend = CameraFactory.create(self._settings)
- self._backend.open()
- except Exception as exc: # pragma: no cover - device specific
- self.error_occurred.emit(str(exc))
+
+ # Initialize camera
+ if not self._initialize_camera():
self.finished.emit()
return
self.started.emit(self._settings)
+ consecutive_errors = 0
+ reconnect_attempts = 0
+
while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
- except TimeoutError:
+
+ # Validate frame size
+ if not self._validate_frame_size(frame):
+ consecutive_errors += 1
+ LOGGER.warning(f"Frame size validation failed ({consecutive_errors}/{self._max_consecutive_errors})")
+ if consecutive_errors >= self._max_consecutive_errors:
+ self.error_occurred.emit("Too many frames with incorrect size")
+ break
+ time.sleep(self._retry_delay)
+ continue
+
+ consecutive_errors = 0 # Reset error count on success
+ reconnect_attempts = 0 # Reset reconnect attempts on success
+
+ except TimeoutError as exc:
+ consecutive_errors += 1
+ LOGGER.warning(f"Camera frame timeout ({consecutive_errors}/{self._max_consecutive_errors}): {exc}")
+
if self._stop_event.is_set():
break
- continue
- except Exception as exc: # pragma: no cover - device specific
- if not self._stop_event.is_set():
- self.error_occurred.emit(str(exc))
- break
+
+ # Handle timeout with retry logic
+ if consecutive_errors < self._max_consecutive_errors:
+ self.warning_occurred.emit(f"Frame timeout (retry {consecutive_errors}/{self._max_consecutive_errors})")
+ time.sleep(self._retry_delay)
+ continue
+ else:
+ # Too many consecutive errors, try to reconnect
+ LOGGER.error(f"Too many consecutive timeouts, attempting reconnection...")
+ if self._attempt_reconnection():
+ consecutive_errors = 0
+ reconnect_attempts += 1
+ self.warning_occurred.emit(f"Camera reconnected (attempt {reconnect_attempts})")
+ continue
+ else:
+ reconnect_attempts += 1
+ if reconnect_attempts >= self._max_reconnect_attempts:
+ self.error_occurred.emit(f"Camera reconnection failed after {reconnect_attempts} attempts")
+ break
+ else:
+ consecutive_errors = 0 # Reset to try again
+ self.warning_occurred.emit(f"Reconnection attempt {reconnect_attempts} failed, retrying...")
+ time.sleep(self._reconnect_delay)
+ continue
+
+ except Exception as exc:
+ consecutive_errors += 1
+ LOGGER.warning(f"Camera read error ({consecutive_errors}/{self._max_consecutive_errors}): {exc}")
+
+ if self._stop_event.is_set():
+ break
+
+ # Handle general errors with retry logic
+ if consecutive_errors < self._max_consecutive_errors:
+ self.warning_occurred.emit(f"Frame read error (retry {consecutive_errors}/{self._max_consecutive_errors})")
+ time.sleep(self._retry_delay)
+ continue
+ else:
+ # Too many consecutive errors, try to reconnect
+ LOGGER.error(f"Too many consecutive errors, attempting reconnection...")
+ if self._attempt_reconnection():
+ consecutive_errors = 0
+ reconnect_attempts += 1
+ self.warning_occurred.emit(f"Camera reconnected (attempt {reconnect_attempts})")
+ continue
+ else:
+ reconnect_attempts += 1
+ if reconnect_attempts >= self._max_reconnect_attempts:
+ self.error_occurred.emit(f"Camera failed after {reconnect_attempts} reconnection attempts: {exc}")
+ break
+ else:
+ consecutive_errors = 0 # Reset to try again
+ self.warning_occurred.emit(f"Reconnection attempt {reconnect_attempts} failed, retrying...")
+ time.sleep(self._reconnect_delay)
+ continue
+
if self._stop_event.is_set():
break
+
self.frame_captured.emit(FrameData(frame, timestamp))
+ # Cleanup
+ self._cleanup_camera()
+ self.finished.emit()
+
+ def _initialize_camera(self) -> bool:
+ """Initialize the camera backend. Returns True on success, False on failure."""
+ try:
+ self._backend = CameraFactory.create(self._settings)
+ self._backend.open()
+ # Don't set expected frame size - will be established from first frame
+ self._expected_frame_size = None
+ LOGGER.info("Camera initialized successfully, frame size will be determined from camera")
+ return True
+ except Exception as exc:
+ LOGGER.exception("Failed to initialize camera", exc_info=exc)
+ self.error_occurred.emit(f"Failed to initialize camera: {exc}")
+ return False
+
+ def _validate_frame_size(self, frame: np.ndarray) -> bool:
+ """Validate that the frame has the expected size. Returns True if valid."""
+ if frame is None or frame.size == 0:
+ LOGGER.warning("Received empty frame")
+ return False
+
+ actual_size = (frame.shape[0], frame.shape[1]) # (height, width)
+
+ if self._expected_frame_size is None:
+ # First frame - establish expected size
+ self._expected_frame_size = actual_size
+ LOGGER.info(f"Established expected frame size: (h={actual_size[0]}, w={actual_size[1]})")
+ return True
+
+ if actual_size != self._expected_frame_size:
+ LOGGER.warning(
+ f"Frame size mismatch: expected (h={self._expected_frame_size[0]}, w={self._expected_frame_size[1]}), "
+ f"got (h={actual_size[0]}, w={actual_size[1]}). Camera may have reconnected with different resolution."
+ )
+ # Update expected size for future frames after reconnection
+ self._expected_frame_size = actual_size
+ LOGGER.info(f"Updated expected frame size to: (h={actual_size[0]}, w={actual_size[1]})")
+ # Emit warning so GUI can restart recording if needed
+ self.warning_occurred.emit(
+ f"Camera resolution changed to {actual_size[1]}x{actual_size[0]}"
+ )
+ return True # Accept the new size
+
+ return True
+
+ def _attempt_reconnection(self) -> bool:
+ """Attempt to reconnect to the camera. Returns True on success, False on failure."""
+ if self._stop_event.is_set():
+ return False
+
+ LOGGER.info("Attempting camera reconnection...")
+
+ # Close existing connection
+ self._cleanup_camera()
+
+ # Wait longer before reconnecting to let the device fully release
+ LOGGER.info(f"Waiting {self._reconnect_delay}s before reconnecting...")
+ time.sleep(self._reconnect_delay)
+
+ if self._stop_event.is_set():
+ return False
+
+ # Try to reinitialize (this will also reset expected frame size)
+ try:
+ self._backend = CameraFactory.create(self._settings)
+ self._backend.open()
+ # Reset expected frame size - will be re-established on first frame
+ self._expected_frame_size = None
+ LOGGER.info("Camera reconnection successful, frame size will be determined from camera")
+ return True
+ except Exception as exc:
+ LOGGER.warning(f"Camera reconnection failed: {exc}")
+ return False
+
+ def _cleanup_camera(self) -> None:
+ """Clean up camera backend resources."""
if self._backend is not None:
try:
self._backend.close()
- except Exception as exc: # pragma: no cover - device specific
- self.error_occurred.emit(str(exc))
+ except Exception as exc:
+ LOGGER.warning(f"Error closing camera: {exc}")
self._backend = None
- self.finished.emit()
@pyqtSlot()
def stop(self) -> None:
@@ -88,6 +250,7 @@ class CameraController(QObject):
started = pyqtSignal(object)
stopped = pyqtSignal()
error = pyqtSignal(str)
+ warning = pyqtSignal(str)
def __init__(self) -> None:
super().__init__()
@@ -133,6 +296,7 @@ def _start_worker(self, settings: CameraSettings) -> None:
self._worker.frame_captured.connect(self.frame_ready)
self._worker.started.connect(self.started)
self._worker.error_occurred.connect(self.error)
+ self._worker.warning_occurred.connect(self.warning)
self._worker.finished.connect(self._thread.quit)
self._worker.finished.connect(self._worker.deleteLater)
self._thread.finished.connect(self._cleanup)
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index 1dc33d8..2e937fd 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -76,8 +76,6 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
settings = CameraSettings(
name=f"Probe {index}",
index=index,
- width=640,
- height=480,
fps=30.0,
backend=backend,
properties={},
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index dc6ce7e..ace3f82 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -120,7 +120,7 @@ def read(self) -> Tuple[np.ndarray, float]:
except ValueError:
frame = array.copy()
except HarvesterTimeoutError as exc:
- raise TimeoutError(str(exc)) from exc
+ raise TimeoutError(str(exc)+ " (GenTL timeout)") from exc
frame = self._convert_frame(frame)
timestamp = time.time()
@@ -244,22 +244,9 @@ def _configure_pixel_format(self, node_map) -> None:
pass
def _configure_resolution(self, node_map) -> None:
- width = int(self.settings.width)
- height = int(self.settings.height)
- if self._rotate in (90, 270):
- width, height = height, width
- try:
- node_map.Width.value = self._adjust_to_increment(
- width, node_map.Width.min, node_map.Width.max, node_map.Width.inc
- )
- except Exception:
- pass
- try:
- node_map.Height.value = self._adjust_to_increment(
- height, node_map.Height.min, node_map.Height.max, node_map.Height.inc
- )
- except Exception:
- pass
+ # Don't configure width/height - use camera's native resolution
+ # Width and height will be determined from actual frames
+ pass
def _configure_exposure(self, node_map) -> None:
if self._exposure is None:
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index cbadf73..ca043bf 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -29,20 +29,43 @@ def open(self) -> None:
def read(self) -> Tuple[np.ndarray, float]:
if self._capture is None:
raise RuntimeError("Camera has not been opened")
- success, frame = self._capture.read()
- if not success:
- raise RuntimeError("Failed to read frame from OpenCV camera")
+
+ # Try grab first - this is non-blocking and helps detect connection issues faster
+ grabbed = self._capture.grab()
+ if not grabbed:
+ # Check if camera is still opened - if not, it's a serious error
+ if not self._capture.isOpened():
+ raise RuntimeError("OpenCV camera connection lost")
+ # Otherwise treat as temporary frame read failure (timeout-like)
+ raise TimeoutError("Failed to grab frame from OpenCV camera (temporary)")
+
+ # Now retrieve the frame
+ success, frame = self._capture.retrieve()
+ if not success or frame is None:
+ raise TimeoutError("Failed to retrieve frame from OpenCV camera (temporary)")
+
return frame, time.time()
def close(self) -> None:
if self._capture is not None:
- self._capture.release()
- self._capture = None
+ try:
+ # Try to release properly
+ self._capture.release()
+ except Exception:
+ pass
+ finally:
+ self._capture = None
+ # Give the system a moment to fully release the device
+ time.sleep(0.1)
def stop(self) -> None:
if self._capture is not None:
- self._capture.release()
- self._capture = None
+ try:
+ self._capture.release()
+ except Exception:
+ pass
+ finally:
+ self._capture = None
def device_name(self) -> str:
base_name = "OpenCV"
@@ -58,9 +81,11 @@ def device_name(self) -> str:
def _configure_capture(self) -> None:
if self._capture is None:
return
- self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, float(self.settings.width))
- self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, float(self.settings.height))
- self._capture.set(cv2.CAP_PROP_FPS, float(self.settings.fps))
+ # Don't set width/height - capture at camera's native resolution
+ # Only set FPS if specified
+ if self.settings.fps:
+ self._capture.set(cv2.CAP_PROP_FPS, float(self.settings.fps))
+ # Set any additional properties from the properties dict
for prop, value in self.settings.properties.items():
if prop == "api":
continue
@@ -69,13 +94,8 @@ def _configure_capture(self) -> None:
except (TypeError, ValueError):
continue
self._capture.set(prop_id, float(value))
- actual_width = self._capture.get(cv2.CAP_PROP_FRAME_WIDTH)
- actual_height = self._capture.get(cv2.CAP_PROP_FRAME_HEIGHT)
+ # Update actual FPS from camera
actual_fps = self._capture.get(cv2.CAP_PROP_FPS)
- if actual_width:
- self.settings.width = int(actual_width)
- if actual_height:
- self.settings.height = int(actual_height)
if actual_fps:
self.settings.fps = float(actual_fps)
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index 72e3f80..c9be6a4 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -13,23 +13,33 @@ class CameraSettings:
name: str = "Camera 0"
index: int = 0
- width: int = 640
- height: int = 480
fps: float = 25.0
backend: str = "gentl"
exposure: int = 500 # 0 = auto, otherwise microseconds
gain: float = 10 # 0.0 = auto, otherwise gain value
+ crop_x0: int = 0 # Left edge of crop region (0 = no crop)
+ crop_y0: int = 0 # Top edge of crop region (0 = no crop)
+ crop_x1: int = 0 # Right edge of crop region (0 = no crop)
+ crop_y1: int = 0 # Bottom edge of crop region (0 = no crop)
properties: Dict[str, Any] = field(default_factory=dict)
def apply_defaults(self) -> "CameraSettings":
- """Ensure width, height and fps are positive numbers."""
+ """Ensure fps is a positive number and validate crop settings."""
- self.width = int(self.width) if self.width else 640
- self.height = int(self.height) if self.height else 480
self.fps = float(self.fps) if self.fps else 30.0
self.exposure = int(self.exposure) if self.exposure else 0
self.gain = float(self.gain) if self.gain else 0.0
+ self.crop_x0 = max(0, int(self.crop_x0)) if hasattr(self, 'crop_x0') else 0
+ self.crop_y0 = max(0, int(self.crop_y0)) if hasattr(self, 'crop_y0') else 0
+ self.crop_x1 = max(0, int(self.crop_x1)) if hasattr(self, 'crop_x1') else 0
+ self.crop_y1 = max(0, int(self.crop_y1)) if hasattr(self, 'crop_y1') else 0
return self
+
+ def get_crop_region(self) -> Optional[tuple[int, int, int, int]]:
+ """Get crop region as (x0, y0, x1, y1) or None if no cropping."""
+ if self.crop_x0 == 0 and self.crop_y0 == 0 and self.crop_x1 == 0 and self.crop_y1 == 0:
+ return None
+ return (self.crop_x0, self.crop_y0, self.crop_x1, self.crop_y1)
@dataclass
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 201f53c..9dd69ca 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -137,7 +137,7 @@ def _start_worker(self, init_frame: np.ndarray, init_timestamp: float) -> None:
if self._worker_thread is not None and self._worker_thread.is_alive():
return
- self._queue = queue.Queue(maxsize=5)
+ self._queue = queue.Queue(maxsize=2)
self._stop_event.clear()
self._worker_thread = threading.Thread(
target=self._worker_loop,
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 0328222..5c76269 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -171,14 +171,6 @@ def _build_camera_group(self) -> QGroupBox:
index_layout.addWidget(self.refresh_cameras_button)
form.addRow("Camera", index_layout)
- self.camera_width = QSpinBox()
- self.camera_width.setRange(1, 7680)
- form.addRow("Width", self.camera_width)
-
- self.camera_height = QSpinBox()
- self.camera_height.setRange(1, 4320)
- form.addRow("Height", self.camera_height)
-
self.camera_fps = QDoubleSpinBox()
self.camera_fps.setRange(1.0, 240.0)
self.camera_fps.setDecimals(2)
@@ -198,6 +190,34 @@ def _build_camera_group(self) -> QGroupBox:
self.camera_gain.setDecimals(2)
form.addRow("Gain", self.camera_gain)
+ # Crop settings
+ crop_layout = QHBoxLayout()
+ self.crop_x0 = QSpinBox()
+ self.crop_x0.setRange(0, 7680)
+ self.crop_x0.setPrefix("x0:")
+ self.crop_x0.setSpecialValueText("x0:None")
+ crop_layout.addWidget(self.crop_x0)
+
+ self.crop_y0 = QSpinBox()
+ self.crop_y0.setRange(0, 4320)
+ self.crop_y0.setPrefix("y0:")
+ self.crop_y0.setSpecialValueText("y0:None")
+ crop_layout.addWidget(self.crop_y0)
+
+ self.crop_x1 = QSpinBox()
+ self.crop_x1.setRange(0, 7680)
+ self.crop_x1.setPrefix("x1:")
+ self.crop_x1.setSpecialValueText("x1:None")
+ crop_layout.addWidget(self.crop_x1)
+
+ self.crop_y1 = QSpinBox()
+ self.crop_y1.setRange(0, 4320)
+ self.crop_y1.setPrefix("y1:")
+ self.crop_y1.setSpecialValueText("y1:None")
+ crop_layout.addWidget(self.crop_y1)
+
+ form.addRow("Crop (x0,y0,x1,y1)", crop_layout)
+
self.camera_properties_edit = QPlainTextEdit()
self.camera_properties_edit.setPlaceholderText(
'{"other_property": "value"}'
@@ -329,6 +349,7 @@ def _connect_signals(self) -> None:
self.camera_controller.frame_ready.connect(self._on_frame_ready)
self.camera_controller.started.connect(self._on_camera_started)
self.camera_controller.error.connect(self._show_error)
+ self.camera_controller.warning.connect(self._show_warning)
self.camera_controller.stopped.connect(self._on_camera_stopped)
self.dlc_processor.pose_ready.connect(self._on_pose_ready)
@@ -338,14 +359,18 @@ def _connect_signals(self) -> None:
# ------------------------------------------------------------------ config
def _apply_config(self, config: ApplicationSettings) -> None:
camera = config.camera
- self.camera_width.setValue(int(camera.width))
- self.camera_height.setValue(int(camera.height))
self.camera_fps.setValue(float(camera.fps))
# Set exposure and gain from config
self.camera_exposure.setValue(int(camera.exposure))
self.camera_gain.setValue(float(camera.gain))
+ # Set crop settings from config
+ self.crop_x0.setValue(int(camera.crop_x0) if hasattr(camera, 'crop_x0') else 0)
+ self.crop_y0.setValue(int(camera.crop_y0) if hasattr(camera, 'crop_y0') else 0)
+ self.crop_x1.setValue(int(camera.crop_x1) if hasattr(camera, 'crop_x1') else 0)
+ self.crop_y1.setValue(int(camera.crop_y1) if hasattr(camera, 'crop_y1') else 0)
+
backend_name = camera.backend or "opencv"
index = self.camera_backend.findData(backend_name)
if index >= 0:
@@ -408,6 +433,12 @@ def _camera_settings_from_ui(self) -> CameraSettings:
exposure = self.camera_exposure.value()
gain = self.camera_gain.value()
+ # Get crop settings from UI
+ crop_x0 = self.crop_x0.value()
+ crop_y0 = self.crop_y0.value()
+ crop_x1 = self.crop_x1.value()
+ crop_y1 = self.crop_y1.value()
+
# Also add to properties dict for backward compatibility with camera backends
if exposure > 0:
properties["exposure"] = exposure
@@ -418,12 +449,14 @@ def _camera_settings_from_ui(self) -> CameraSettings:
settings = CameraSettings(
name=name_text or f"Camera {index}",
index=index,
- width=self.camera_width.value(),
- height=self.camera_height.value(),
fps=self.camera_fps.value(),
backend=backend_text or "opencv",
exposure=exposure,
gain=gain,
+ crop_x0=crop_x0,
+ crop_y0=crop_y0,
+ crop_x1=crop_x1,
+ crop_y1=crop_y1,
properties=properties,
)
return settings.apply_defaults()
@@ -441,7 +474,7 @@ def _refresh_camera_indices(
backend = self._current_backend_name()
detected = CameraFactory.detect_cameras(backend)
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
- print(
+ logging.info(
f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
)
self._detected_cameras = detected
@@ -500,7 +533,7 @@ def _refresh_camera_indices(
backend = self._current_backend_name()
detected = CameraFactory.detect_cameras(backend)
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
- print(
+ logging.info(
f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
)
self._detected_cameras = detected
@@ -687,23 +720,17 @@ def _on_camera_started(self, settings: CameraSettings) -> None:
self._active_camera_settings = settings
self.preview_button.setEnabled(False)
self.stop_preview_button.setEnabled(True)
- self.camera_width.blockSignals(True)
- self.camera_width.setValue(int(settings.width))
- self.camera_width.blockSignals(False)
- self.camera_height.blockSignals(True)
- self.camera_height.setValue(int(settings.height))
- self.camera_height.blockSignals(False)
if getattr(settings, "fps", None):
self.camera_fps.blockSignals(True)
self.camera_fps.setValue(float(settings.fps))
self.camera_fps.blockSignals(False)
- resolution = f"{int(settings.width)}×{int(settings.height)}"
+ # Resolution will be determined from actual camera frames
if getattr(settings, "fps", None):
fps_text = f"{float(settings.fps):.2f} FPS"
else:
fps_text = "unknown FPS"
self.statusBar().showMessage(
- f"Camera preview started: {resolution} @ {fps_text}", 5000
+ f"Camera preview started @ {fps_text}", 5000
)
self._update_inference_buttons()
self._update_camera_controls_enabled()
@@ -758,11 +785,13 @@ def _update_camera_controls_enabled(self) -> None:
self.camera_backend,
self.camera_index,
self.refresh_cameras_button,
- self.camera_width,
- self.camera_height,
self.camera_fps,
self.camera_exposure,
self.camera_gain,
+ self.crop_x0,
+ self.crop_y0,
+ self.crop_x1,
+ self.crop_y1,
self.camera_properties_edit,
self.rotation_combo,
self.codec_combo,
@@ -924,7 +953,7 @@ def _start_recording(self) -> None:
output_path = recording.output_path()
self._video_recorder = VideoRecorder(
output_path,
- frame_size=(int(width), int(height)),
+ frame_size=(height, width), # Use numpy convention: (height, width)
frame_rate=float(frame_rate),
codec=recording.codec,
crf=recording.crf,
@@ -974,17 +1003,18 @@ def _stop_recording(self) -> None:
def _on_frame_ready(self, frame_data: FrameData) -> None:
raw_frame = frame_data.image
self._raw_frame = raw_frame
- frame = self._apply_rotation(raw_frame)
+
+ # Apply cropping before rotation
+ frame = self._apply_crop(raw_frame)
+
+ # Apply rotation
+ frame = self._apply_rotation(frame)
frame = np.ascontiguousarray(frame)
self._current_frame = frame
self._track_camera_frame()
- if self._active_camera_settings is not None:
- height, width = frame.shape[:2]
- self._active_camera_settings.width = int(width)
- self._active_camera_settings.height = int(height)
if self._video_recorder and self._video_recorder.is_running:
try:
- success = self._video_recorder.write(frame)
+ success = self._video_recorder.write(frame, timestamp=frame_data.timestamp)
if not success:
now = time.perf_counter()
if now - self._last_drop_warning > 1.0:
@@ -993,8 +1023,19 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
)
self._last_drop_warning = now
except RuntimeError as exc:
- self._show_error(str(exc))
- self._stop_recording()
+ # Check if it's a frame size error
+ if "Frame size changed" in str(exc):
+ self._show_warning(f"Camera resolution changed - restarting recording: {exc}")
+ self._stop_recording()
+ # Restart recording with new resolution if enabled
+ if self.recording_enabled_checkbox.isChecked():
+ try:
+ self._start_recording()
+ except Exception as restart_exc:
+ self._show_error(f"Failed to restart recording: {restart_exc}")
+ else:
+ self._show_error(str(exc))
+ self._stop_recording()
if self._dlc_active:
self.dlc_processor.enqueue_frame(frame, frame_data.timestamp)
self._display_frame(frame)
@@ -1003,7 +1044,7 @@ def _on_pose_ready(self, result: PoseResult) -> None:
if not self._dlc_active:
return
self._last_pose = result
- logging.info(f"Pose result: {result.pose}, Timestamp: {result.timestamp}")
+ #logging.info(f"Pose result: {result.pose}, Timestamp: {result.timestamp}")
if self._current_frame is not None:
self._display_frame(self._current_frame, force=True)
@@ -1025,6 +1066,31 @@ def _update_video_display(self, frame: np.ndarray) -> None:
image = QImage(rgb.data, w, h, bytes_per_line, QImage.Format.Format_RGB888)
self.video_label.setPixmap(QPixmap.fromImage(image))
+ def _apply_crop(self, frame: np.ndarray) -> np.ndarray:
+ """Apply cropping to the frame based on settings."""
+ if self._active_camera_settings is None:
+ return frame
+
+ crop_region = self._active_camera_settings.get_crop_region()
+ if crop_region is None:
+ return frame
+
+ x0, y0, x1, y1 = crop_region
+ height, width = frame.shape[:2]
+
+ # Validate and constrain crop coordinates
+ x0 = max(0, min(x0, width))
+ y0 = max(0, min(y0, height))
+ x1 = max(x0, min(x1, width)) if x1 > 0 else width
+ y1 = max(y0, min(y1, height)) if y1 > 0 else height
+
+ # Apply crop
+ if x0 < x1 and y0 < y1:
+ return frame[y0:y1, x0:x1]
+ else:
+ # Invalid crop region, return original frame
+ return frame
+
def _apply_rotation(self, frame: np.ndarray) -> np.ndarray:
if self._rotation_degrees == 90:
return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
@@ -1070,6 +1136,10 @@ def _show_error(self, message: str) -> None:
self.statusBar().showMessage(message, 5000)
QMessageBox.critical(self, "Error", message)
+ def _show_warning(self, message: str) -> None:
+ """Display a warning message in the status bar without blocking."""
+ self.statusBar().showMessage(f"⚠ {message}", 3000)
+
# ------------------------------------------------------------------ Qt overrides
def closeEvent(self, event: QCloseEvent) -> None: # pragma: no cover - GUI behaviour
if self.camera_controller.is_running():
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index d729b02..2190314 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -1,6 +1,7 @@
"""Video recording support using the vidgear library."""
from __future__ import annotations
+import json
import logging
import queue
import threading
@@ -8,7 +9,7 @@
from collections import deque
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Dict, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
import numpy as np
@@ -69,6 +70,7 @@ def __init__(
self._written_times: deque[float] = deque(maxlen=600)
self._encode_error: Optional[Exception] = None
self._last_log_time = 0.0
+ self._frame_timestamps: List[float] = []
@property
def is_running(self) -> bool:
@@ -85,7 +87,7 @@ def start(self) -> None:
writer_kwargs: Dict[str, Any] = {
"compression_mode": True,
- "logging": True,
+ "logging": False,
"-input_framerate": fps_value,
"-vcodec": (self._codec or "libx264").strip() or "libx264",
"-crf": int(self._crf),
@@ -101,6 +103,7 @@ def start(self) -> None:
self._total_latency = 0.0
self._last_latency = 0.0
self._written_times.clear()
+ self._frame_timestamps.clear()
self._encode_error = None
self._stop_event.clear()
self._writer_thread = threading.Thread(
@@ -116,12 +119,20 @@ def configure_stream(
self._frame_size = frame_size
self._frame_rate = frame_rate
- def write(self, frame: np.ndarray) -> bool:
+ def write(self, frame: np.ndarray, timestamp: Optional[float] = None) -> bool:
if not self.is_running or self._queue is None:
return False
error = self._current_error()
if error is not None:
raise RuntimeError(f"Video encoding failed: {error}") from error
+
+ # Record timestamp for this frame
+ if timestamp is None:
+ timestamp = time.time()
+ with self._stats_lock:
+ self._frame_timestamps.append(timestamp)
+
+ # Convert frame to uint8 if needed
if frame.dtype != np.uint8:
frame_float = frame.astype(np.float32, copy=False)
max_val = float(frame_float.max()) if frame_float.size else 0.0
@@ -129,9 +140,31 @@ def write(self, frame: np.ndarray) -> bool:
if max_val > 0:
scale = 255.0 / max_val if max_val > 255.0 else (255.0 if max_val <= 1.0 else 1.0)
frame = np.clip(frame_float * scale, 0.0, 255.0).astype(np.uint8)
+
+ # Convert grayscale to RGB if needed
if frame.ndim == 2:
frame = np.repeat(frame[:, :, None], 3, axis=2)
+
+ # Ensure contiguous array
frame = np.ascontiguousarray(frame)
+
+ # Check if frame size matches expected size
+ if self._frame_size is not None:
+ expected_h, expected_w = self._frame_size
+ actual_h, actual_w = frame.shape[:2]
+ if (actual_h, actual_w) != (expected_h, expected_w):
+ logger.warning(
+ f"Frame size mismatch: expected (h={expected_h}, w={expected_w}), "
+ f"got (h={actual_h}, w={actual_w}). "
+ "Stopping recorder to prevent encoding errors."
+ )
+ # Set error to stop recording gracefully
+ with self._stats_lock:
+ self._encode_error = ValueError(
+ f"Frame size changed from (h={expected_h}, w={expected_w}) to (h={actual_h}, w={actual_w})"
+ )
+ return False
+
try:
assert self._queue is not None
self._queue.put(frame, block=False)
@@ -167,6 +200,10 @@ def stop(self) -> None:
self._writer.close()
except Exception:
logger.exception("Failed to close WriteGear cleanly")
+
+ # Save timestamps to JSON file
+ self._save_timestamps()
+
self._writer = None
self._writer_thread = None
self._queue = None
@@ -239,12 +276,12 @@ def _writer_loop(self) -> None:
if now - self._last_log_time >= 1.0:
fps = self._compute_write_fps_locked()
queue_size = self._queue.qsize()
- logger.info(
- "Recorder throughput: %.2f fps, latency %.2f ms, queue=%d",
- fps,
- elapsed * 1000.0,
- queue_size,
- )
+ # logger.info(
+ # "Recorder throughput: %.2f fps, latency %.2f ms, queue=%d",
+ # fps,
+ # elapsed * 1000.0,
+ # queue_size,
+ # )
self._last_log_time = now
self._queue.task_done()
self._finalize_writer()
@@ -269,3 +306,34 @@ def _compute_write_fps_locked(self) -> float:
def _current_error(self) -> Optional[Exception]:
with self._stats_lock:
return self._encode_error
+
+ def _save_timestamps(self) -> None:
+ """Save frame timestamps to a JSON file alongside the video."""
+ if not self._frame_timestamps:
+ logger.info("No timestamps to save")
+ return
+
+ # Create timestamps file path
+ timestamp_file = self._output.with_suffix('').with_suffix(self._output.suffix + '_timestamps.json')
+
+ try:
+ with self._stats_lock:
+ timestamps = self._frame_timestamps.copy()
+
+ # Prepare metadata
+ data = {
+ "video_file": str(self._output.name),
+ "num_frames": len(timestamps),
+ "timestamps": timestamps,
+ "start_time": timestamps[0] if timestamps else None,
+ "end_time": timestamps[-1] if timestamps else None,
+ "duration_seconds": timestamps[-1] - timestamps[0] if len(timestamps) > 1 else 0.0,
+ }
+
+ # Write to JSON
+ with open(timestamp_file, 'w') as f:
+ json.dump(data, f, indent=2)
+
+ logger.info(f"Saved {len(timestamps)} frame timestamps to {timestamp_file}")
+ except Exception as exc:
+ logger.exception(f"Failed to save timestamps to {timestamp_file}: {exc}")
From 78238299e6e6551b84bc31df496c00f62825d408 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 23 Oct 2025 18:47:08 +0200
Subject: [PATCH 11/26] modified the processor to be controllable
---
dlclivegui/processors/dlc_processor_socket.py | 99 +++++++--
example_recording_control.py | 194 ++++++++++++++++++
2 files changed, 276 insertions(+), 17 deletions(-)
create mode 100644 example_recording_control.py
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index bd183af..dbb0f90 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -131,9 +131,27 @@ def __init__(
self.pose_time = deque()
self.original_pose = deque()
+ self._session_name = "test_session"
+ self.filename = None
+ self._recording = Event() # Thread-safe recording flag
+
# State
self.curr_step = 0
self.save_original = save_original
+
+ @property
+ def recording(self):
+ """Thread-safe recording flag."""
+ return self._recording.is_set()
+
+ @property
+ def session_name(self):
+ return self._session_name
+
+ @session_name.setter
+ def session_name(self, name):
+ self._session_name = name
+ self.filename = f"{name}_dlc_processor_data.pkl"
def _accept_loop(self):
"""Background thread to accept new client connections."""
@@ -154,7 +172,8 @@ def _rx_loop(self, c):
try:
if c.poll(0.05):
msg = c.recv()
- # Optional: handle client messages here
+ # Handle control messages from client
+ self._handle_client_message(msg)
except (EOFError, OSError, BrokenPipeError):
break
try:
@@ -163,6 +182,42 @@ def _rx_loop(self, c):
pass
self.conns.discard(c)
LOG.info("Client disconnected")
+
+ def _handle_client_message(self, msg):
+ """Handle control messages from clients."""
+ if not isinstance(msg, dict):
+ return
+
+ cmd = msg.get("cmd")
+ if cmd == "set_session_name":
+ session_name = msg.get("session_name", "default_session")
+ self.session_name = session_name
+ LOG.info(f"Session name set to: {session_name}")
+
+ elif cmd == "start_recording":
+ self._recording.set()
+ # Clear all data queues
+ self._clear_data_queues()
+ self.curr_step = 0
+ LOG.info("Recording started, data queues cleared")
+
+ elif cmd == "stop_recording":
+ self._recording.clear()
+ LOG.info("Recording stopped")
+
+ elif cmd == "save":
+ filename = msg.get("filename", self.filename)
+ save_code = self.save(filename)
+ LOG.info(f"Save {'successful' if save_code == 1 else 'failed'}: {filename}")
+
+ def _clear_data_queues(self):
+ """Clear all data storage queues. Override in subclasses to clear additional queues."""
+ self.time_stamp.clear()
+ self.step.clear()
+ self.frame_time.clear()
+ self.pose_time.clear()
+ if self.save_original:
+ self.original_pose.clear()
def broadcast(self, payload):
"""Send payload to all connected clients."""
@@ -202,12 +257,13 @@ def process(self, pose, **kwargs):
# Update step counter
self.curr_step = self.curr_step + 1
- # Store metadata
- self.time_stamp.append(curr_time)
- self.step.append(self.curr_step)
- self.frame_time.append(kwargs.get("frame_time", -1))
- if "pose_time" in kwargs:
- self.pose_time.append(kwargs["pose_time"])
+ # Store metadata (only if recording)
+ if self.recording:
+ self.time_stamp.append(curr_time)
+ self.step.append(self.curr_step)
+ self.frame_time.append(kwargs.get("frame_time", -1))
+ if "pose_time" in kwargs:
+ self.pose_time.append(kwargs["pose_time"])
# Broadcast raw pose to all connected clients
payload = [curr_time, pose]
@@ -344,6 +400,14 @@ def __init__(
self.filter_kwargs = filter_kwargs or {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0}
self.filters = None # Will be initialized on first pose
+ def _clear_data_queues(self):
+ """Clear all data storage queues including pose-specific ones."""
+ super()._clear_data_queues()
+ self.center_x.clear()
+ self.center_y.clear()
+ self.heading_direction.clear()
+ self.head_angle.clear()
+
def _initialize_filters(self, vals):
"""Initialize One-Euro filters for each output variable."""
t0 = self.timing_func()
@@ -423,16 +487,17 @@ def process(self, pose, **kwargs):
# Update step counter
self.curr_step = self.curr_step + 1
- # Store processed data
- self.center_x.append(vals[0])
- self.center_y.append(vals[1])
- self.heading_direction.append(vals[2])
- self.head_angle.append(vals[3])
- self.time_stamp.append(curr_time)
- self.step.append(self.curr_step)
- self.frame_time.append(kwargs.get("frame_time", -1))
- if "pose_time" in kwargs:
- self.pose_time.append(kwargs["pose_time"])
+ # Store processed data (only if recording)
+ if self.recording:
+ self.center_x.append(vals[0])
+ self.center_y.append(vals[1])
+ self.heading_direction.append(vals[2])
+ self.head_angle.append(vals[3])
+ self.time_stamp.append(curr_time)
+ self.step.append(self.curr_step)
+ self.frame_time.append(kwargs.get("frame_time", -1))
+ if "pose_time" in kwargs:
+ self.pose_time.append(kwargs["pose_time"])
# Broadcast processed values to all connected clients
payload = [curr_time, vals[0], vals[1], vals[2], vals[3]]
diff --git a/example_recording_control.py b/example_recording_control.py
new file mode 100644
index 0000000..ed426e8
--- /dev/null
+++ b/example_recording_control.py
@@ -0,0 +1,194 @@
+"""
+Example: Recording control with DLCClient and MyProcessor_socket
+
+This demonstrates:
+1. Starting a processor
+2. Connecting a client
+3. Controlling recording (start/stop/save) from the client
+4. Session name management
+"""
+
+import time
+import sys
+from pathlib import Path
+
+# Add parent directory to path
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from mouse_ar.ctrl.dlc_client import DLCClient
+
+
+def example_recording_workflow():
+ """Complete workflow: processor + client with recording control."""
+
+ print("\n" + "="*70)
+ print("EXAMPLE: Recording Control Workflow")
+ print("="*70)
+
+ # NOTE: This example assumes MyProcessor_socket is already running
+ # Start it separately with:
+ # from dlc_processor_socket import MyProcessor_socket
+ # processor = MyProcessor_socket(bind=("localhost", 6000))
+ # # Then run DLCLive with this processor
+
+ print("\n[CLIENT] Connecting to processor at localhost:6000...")
+ client = DLCClient(address=("localhost", 6000))
+
+ try:
+ # Start the client (connects and begins receiving data)
+ client.start()
+ print("[CLIENT] Connected!")
+ time.sleep(0.5) # Wait for connection to stabilize
+
+ # Set session name
+ print("\n[CLIENT] Setting session name to 'experiment_001'...")
+ client.set_session_name("experiment_001")
+ time.sleep(0.2)
+
+ # Start recording
+ print("[CLIENT] Starting recording (clears processor data queues)...")
+ client.start_recording()
+ time.sleep(0.2)
+
+ # Receive some data
+ print("\n[CLIENT] Receiving data for 5 seconds...")
+ for i in range(5):
+ data = client.read()
+ if data:
+ vals = data["vals"]
+ print(f" t={vals[0]:.2f}, x={vals[1]:.1f}, y={vals[2]:.1f}, "
+ f"heading={vals[3]:.1f}°, head_angle={vals[4]:.2f}rad")
+ time.sleep(1.0)
+
+ # Stop recording
+ print("\n[CLIENT] Stopping recording...")
+ client.stop_recording()
+ time.sleep(0.2)
+
+ # Trigger save
+ print("[CLIENT] Triggering save on processor...")
+ client.trigger_save() # Uses processor's default filename
+ # OR specify custom filename:
+ # client.trigger_save(filename="my_custom_data.pkl")
+ time.sleep(0.5)
+
+ print("\n[CLIENT] ✓ Workflow complete!")
+
+ except Exception as e:
+ print(f"\n[ERROR] {e}")
+ print("\nMake sure MyProcessor_socket is running!")
+ print("Example:")
+ print(" from dlc_processor_socket import MyProcessor_socket")
+ print(" processor = MyProcessor_socket()")
+ print(" # Then run DLCLive with this processor")
+
+ finally:
+ print("\n[CLIENT] Closing connection...")
+ client.close()
+
+
+def example_multiple_sessions():
+ """Example: Recording multiple sessions with the same processor."""
+
+ print("\n" + "="*70)
+ print("EXAMPLE: Multiple Sessions")
+ print("="*70)
+
+ client = DLCClient(address=("localhost", 6000))
+
+ try:
+ client.start()
+ print("[CLIENT] Connected!")
+ time.sleep(0.5)
+
+ # Session 1
+ print("\n--- SESSION 1 ---")
+ client.set_session_name("trial_001")
+ client.start_recording()
+ print("Recording session 'trial_001' for 3 seconds...")
+ time.sleep(3.0)
+ client.stop_recording()
+ client.trigger_save() # Saves as "trial_001_dlc_processor_data.pkl"
+ print("Session 1 saved!")
+
+ time.sleep(1.0)
+
+ # Session 2
+ print("\n--- SESSION 2 ---")
+ client.set_session_name("trial_002")
+ client.start_recording()
+ print("Recording session 'trial_002' for 3 seconds...")
+ time.sleep(3.0)
+ client.stop_recording()
+ client.trigger_save() # Saves as "trial_002_dlc_processor_data.pkl"
+ print("Session 2 saved!")
+
+ print("\n✓ Multiple sessions recorded successfully!")
+
+ except Exception as e:
+ print(f"\n[ERROR] {e}")
+
+ finally:
+ client.close()
+
+
+def example_command_api():
+ """Example: Using the low-level command API."""
+
+ print("\n" + "="*70)
+ print("EXAMPLE: Low-level Command API")
+ print("="*70)
+
+ client = DLCClient(address=("localhost", 6000))
+
+ try:
+ client.start()
+ time.sleep(0.5)
+
+ # Using send_command directly
+ print("\n[CLIENT] Using send_command()...")
+
+ # Set session name
+ client.send_command("set_session_name", session_name="custom_session")
+ print(" ✓ Sent: set_session_name")
+ time.sleep(0.2)
+
+ # Start recording
+ client.send_command("start_recording")
+ print(" ✓ Sent: start_recording")
+ time.sleep(2.0)
+
+ # Stop recording
+ client.send_command("stop_recording")
+ print(" ✓ Sent: stop_recording")
+ time.sleep(0.2)
+
+ # Save with custom filename
+ client.send_command("save", filename="my_data.pkl")
+ print(" ✓ Sent: save")
+ time.sleep(0.5)
+
+ print("\n✓ Commands sent successfully!")
+
+ except Exception as e:
+ print(f"\n[ERROR] {e}")
+
+ finally:
+ client.close()
+
+
+if __name__ == "__main__":
+ print("\n" + "="*70)
+ print("DLC PROCESSOR RECORDING CONTROL EXAMPLES")
+ print("="*70)
+ print("\nNOTE: These examples require MyProcessor_socket to be running.")
+ print("Start it separately before running these examples.")
+ print("="*70)
+
+ # Uncomment the example you want to run:
+
+ # example_recording_workflow()
+ # example_multiple_sessions()
+ # example_command_api()
+
+ print("\nUncomment an example in the script to run it.")
From 184e87b0b2672fd67cdf8984c4959b5f42f0564b Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 24 Oct 2025 15:01:07 +0200
Subject: [PATCH 12/26] Add GenTL device count retrieval, bounding box
settings, and processor integration
- Implemented `get_device_count` method in `GenTLCameraBackend` to retrieve the number of GenTL devices detected.
- Added `max_devices` configuration option in `CameraSettings` to limit device probing.
- Introduced `BoundingBoxSettings` for bounding box visualization, integrated into the main GUI.
- Enhanced `DLCLiveProcessor` to accept a processor instance during configuration.
- Updated GUI to support processor selection and auto-recording based on processor commands.
- Refactored camera properties handling and removed deprecated advanced properties editor.
- Improved error handling and logging for processor connections and recording states.
---
dlclivegui/cameras/factory.py | 13 +-
dlclivegui/cameras/gentl_backend.py | 82 +++-
dlclivegui/config.py | 17 +-
dlclivegui/dlc_processor.py | 11 +-
dlclivegui/gui.py | 412 +++++++++++++++---
dlclivegui/processors/dlc_processor_socket.py | 17 +-
example_recording_control.py | 194 ---------
7 files changed, 477 insertions(+), 269 deletions(-)
delete mode 100644 example_recording_control.py
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index 2e937fd..540e352 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -57,6 +57,7 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
The backend identifier, e.g. ``"opencv"``.
max_devices:
Upper bound for the indices that should be probed.
+ For GenTL backend, the actual device count is queried if available.
Returns
-------
@@ -71,8 +72,18 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
if not backend_cls.is_available():
return []
+ # For GenTL backend, try to get actual device count
+ num_devices = max_devices
+ if hasattr(backend_cls, 'get_device_count'):
+ try:
+ actual_count = backend_cls.get_device_count()
+ if actual_count >= 0:
+ num_devices = actual_count
+ except Exception:
+ pass
+
detected: List[DetectedCamera] = []
- for index in range(max_devices):
+ for index in range(num_devices):
settings = CameraSettings(
name=f"Probe {index}",
index=index,
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index ace3f82..943cf4a 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -53,6 +53,36 @@ def __init__(self, settings):
def is_available(cls) -> bool:
return Harvester is not None
+ @classmethod
+ def get_device_count(cls) -> int:
+ """Get the actual number of GenTL devices detected by Harvester.
+
+ Returns the number of devices found, or -1 if detection fails.
+ """
+ if Harvester is None:
+ return -1
+
+ harvester = None
+ try:
+ harvester = Harvester()
+ # Use the static helper to find CTI file with default patterns
+ cti_file = cls._search_cti_file(cls._DEFAULT_CTI_PATTERNS)
+
+ if not cti_file:
+ return -1
+
+ harvester.add_file(cti_file)
+ harvester.update()
+ return len(harvester.device_info_list)
+ except Exception:
+ return -1
+ finally:
+ if harvester is not None:
+ try:
+ harvester.reset()
+ except Exception:
+ pass
+
def open(self) -> None:
if Harvester is None: # pragma: no cover - optional dependency
raise RuntimeError(
@@ -90,6 +120,32 @@ def open(self) -> None:
remote = self._acquirer.remote_device
node_map = remote.node_map
+ #print(dir(node_map))
+ """
+ ['AcquisitionBurstFrameCount', 'AcquisitionControl', 'AcquisitionFrameRate', 'AcquisitionMode',
+ 'AcquisitionStart', 'AcquisitionStop', 'AnalogControl', 'AutoFunctionsROI', 'AutoFunctionsROIEnable',
+ 'AutoFunctionsROIHeight', 'AutoFunctionsROILeft', 'AutoFunctionsROIPreset', 'AutoFunctionsROITop',
+ 'AutoFunctionsROIWidth', 'BinningHorizontal', 'BinningVertical', 'BlackLevel', 'CameraRegisterAddress',
+ 'CameraRegisterAddressSpace', 'CameraRegisterControl', 'CameraRegisterRead', 'CameraRegisterValue',
+ 'CameraRegisterWrite', 'Contrast', 'DecimationHorizontal', 'DecimationVertical', 'Denoise',
+ 'DeviceControl', 'DeviceFirmwareVersion', 'DeviceModelName', 'DeviceReset', 'DeviceSFNCVersionMajor',
+ 'DeviceSFNCVersionMinor', 'DeviceSFNCVersionSubMinor', 'DeviceScanType', 'DeviceSerialNumber',
+ 'DeviceTLType', 'DeviceTLVersionMajor', 'DeviceTLVersionMinor', 'DeviceTLVersionSubMinor',
+ 'DeviceTemperature', 'DeviceTemperatureSelector', 'DeviceType', 'DeviceUserID', 'DeviceVendorName',
+ 'DigitalIO', 'ExposureAuto', 'ExposureAutoHighlightReduction', 'ExposureAutoLowerLimit',
+ 'ExposureAutoReference', 'ExposureAutoUpperLimit', 'ExposureAutoUpperLimitAuto', 'ExposureTime',
+ 'GPIn', 'GPOut', 'Gain', 'GainAuto', 'GainAutoLowerLimit', 'GainAutoUpperLimit', 'Gamma', 'Height',
+ 'HeightMax', 'IMXLowLatencyTriggerMode', 'ImageFormatControl', 'OffsetAutoCenter', 'OffsetX', 'OffsetY',
+ 'PayloadSize', 'PixelFormat', 'ReverseX', 'ReverseY', 'Root', 'SensorHeight', 'SensorWidth', 'Sharpness',
+ 'ShowOverlay', 'SoftwareAnalogControl', 'SoftwareTransformControl', 'SoftwareTransformEnable',
+ 'StrobeDelay', 'StrobeDuration', 'StrobeEnable', 'StrobeOperation', 'StrobePolarity', 'TLParamsLocked',
+ 'TestControl', 'TestPendingAck', 'TimestampLatch', 'TimestampLatchValue', 'TimestampReset', 'ToneMappingAuto',
+ 'ToneMappingControl', 'ToneMappingEnable', 'ToneMappingGlobalBrightness', 'ToneMappingIntensity',
+ 'TransportLayerControl', 'TriggerActivation', 'TriggerDebouncer', 'TriggerDelay', 'TriggerDenoise',
+ 'TriggerMask', 'TriggerMode', 'TriggerOverlap', 'TriggerSelector', 'TriggerSoftware', 'TriggerSource',
+ 'UserSetControl', 'UserSetDefault', 'UserSetLoad', 'UserSetSave', 'UserSetSelector', 'Width', 'WidthMax']
+ """
+
self._device_label = self._resolve_device_label(node_map)
self._configure_pixel_format(node_map)
@@ -172,16 +228,30 @@ def _parse_crop(self, crop) -> Optional[Tuple[int, int, int, int]]:
return tuple(int(v) for v in crop)
return None
- def _find_cti_file(self) -> str:
- patterns: List[str] = list(self._cti_search_paths)
+ @staticmethod
+ def _search_cti_file(patterns: Tuple[str, ...]) -> Optional[str]:
+ """Search for a CTI file using the given patterns.
+
+ Returns the first CTI file found, or None if none found.
+ """
for pattern in patterns:
for file_path in glob.glob(pattern):
if os.path.isfile(file_path):
return file_path
- raise RuntimeError(
- "Could not locate a GenTL producer (.cti) file. Set 'cti_file' in "
- "camera.properties or provide search paths via 'cti_search_paths'."
- )
+ return None
+
+ def _find_cti_file(self) -> str:
+ """Find a CTI file using configured or default search paths.
+
+ Raises RuntimeError if no CTI file is found.
+ """
+ cti_file = self._search_cti_file(self._cti_search_paths)
+ if cti_file is None:
+ raise RuntimeError(
+ "Could not locate a GenTL producer (.cti) file. Set 'cti_file' in "
+ "camera.properties or provide search paths via 'cti_search_paths'."
+ )
+ return cti_file
def _available_serials(self) -> List[str]:
assert self._harvester is not None
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index c9be6a4..ca1f3e5 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -21,6 +21,7 @@ class CameraSettings:
crop_y0: int = 0 # Top edge of crop region (0 = no crop)
crop_x1: int = 0 # Right edge of crop region (0 = no crop)
crop_y1: int = 0 # Bottom edge of crop region (0 = no crop)
+ max_devices: int = 3 # Maximum number of devices to probe during detection
properties: Dict[str, Any] = field(default_factory=dict)
def apply_defaults(self) -> "CameraSettings":
@@ -51,6 +52,17 @@ class DLCProcessorSettings:
model_type: Optional[str] = "base"
+@dataclass
+class BoundingBoxSettings:
+ """Configuration for bounding box visualization."""
+
+ enabled: bool = False
+ x0: int = 0
+ y0: int = 0
+ x1: int = 200
+ y1: int = 100
+
+
@dataclass
class RecordingSettings:
"""Configuration for video recording."""
@@ -94,6 +106,7 @@ class ApplicationSettings:
camera: CameraSettings = field(default_factory=CameraSettings)
dlc: DLCProcessorSettings = field(default_factory=DLCProcessorSettings)
recording: RecordingSettings = field(default_factory=RecordingSettings)
+ bbox: BoundingBoxSettings = field(default_factory=BoundingBoxSettings)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
@@ -108,7 +121,8 @@ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
recording_data = dict(data.get("recording", {}))
recording_data.pop("options", None)
recording = RecordingSettings(**recording_data)
- return cls(camera=camera, dlc=dlc, recording=recording)
+ bbox = BoundingBoxSettings(**data.get("bbox", {}))
+ return cls(camera=camera, dlc=dlc, recording=recording, bbox=bbox)
def to_dict(self) -> Dict[str, Any]:
"""Serialise the configuration to a dictionary."""
@@ -117,6 +131,7 @@ def to_dict(self) -> Dict[str, Any]:
"camera": asdict(self.camera),
"dlc": asdict(self.dlc),
"recording": asdict(self.recording),
+ "bbox": asdict(self.bbox),
}
@classmethod
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 9dd69ca..6358c74 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -54,6 +54,7 @@ def __init__(self) -> None:
super().__init__()
self._settings = DLCProcessorSettings()
self._dlc: Optional[Any] = None
+ self._processor: Optional[Any] = None
self._queue: Optional[queue.Queue[Any]] = None
self._worker_thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
@@ -67,8 +68,9 @@ def __init__(self) -> None:
self._processing_times: deque[float] = deque(maxlen=60)
self._stats_lock = threading.Lock()
- def configure(self, settings: DLCProcessorSettings) -> None:
+ def configure(self, settings: DLCProcessorSettings, processor: Optional[Any] = None) -> None:
self._settings = settings
+ self._processor = processor
def reset(self) -> None:
"""Stop the worker thread and drop the current DLCLive instance."""
@@ -93,6 +95,10 @@ def enqueue_frame(self, frame: np.ndarray, timestamp: float) -> None:
self._start_worker(frame.copy(), timestamp)
return
+ # Don't count dropped frames until processor is initialized
+ if not self._initialized:
+ return
+
if self._queue is not None:
try:
# Non-blocking put - drop frame if queue is full
@@ -178,10 +184,11 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
options = {
"model_path": self._settings.model_path,
"model_type": self._settings.model_type,
- "processor": None,
+ "processor": self._processor,
"dynamic": [False,0.5,10],
"resize": 1.0,
}
+ # todo expose more parameters from settings
self._dlc = DLCLive(**options)
self._dlc.init_inference(init_frame)
self._initialized = True
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 5c76269..66db93c 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -41,12 +41,14 @@
from dlclivegui.cameras.factory import DetectedCamera
from dlclivegui.config import (
ApplicationSettings,
+ BoundingBoxSettings,
CameraSettings,
DLCProcessorSettings,
RecordingSettings,
DEFAULT_CONFIG,
)
from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult, ProcessorStats
+from dlclivegui.processors.processor_utils import scan_processor_folder, instantiate_from_scan
from dlclivegui.video_recorder import RecorderStats, VideoRecorder
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
@@ -76,6 +78,15 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._display_interval = 1.0 / 25.0
self._last_display_time = 0.0
self._dlc_initialized = False
+ self._scanned_processors: dict = {}
+ self._processor_keys: list = []
+ self._last_processor_vid_recording = False
+ self._auto_record_session_name: Optional[str] = None
+ self._bbox_x0 = 0
+ self._bbox_y0 = 0
+ self._bbox_x1 = 0
+ self._bbox_y1 = 0
+ self._bbox_enabled = False
self.camera_controller = CameraController()
self.dlc_processor = DLCLiveProcessor()
@@ -83,6 +94,7 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._setup_ui()
self._connect_signals()
self._apply_config(self._config)
+ self._refresh_processors() # Scan and populate processor dropdown
self._update_inference_buttons()
self._update_camera_controls_enabled()
self._metrics_timer = QTimer(self)
@@ -96,6 +108,7 @@ def _setup_ui(self) -> None:
central = QWidget()
layout = QHBoxLayout(central)
+ # Video display widget
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
@@ -103,27 +116,36 @@ def _setup_ui(self) -> None:
QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding
)
+ # Controls panel with fixed width to prevent shifting
controls_widget = QWidget()
+ controls_widget.setMaximumWidth(500)
+ controls_widget.setSizePolicy(
+ QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding
+ )
controls_layout = QVBoxLayout(controls_widget)
+ controls_layout.setContentsMargins(5, 5, 5, 5)
controls_layout.addWidget(self._build_camera_group())
controls_layout.addWidget(self._build_dlc_group())
controls_layout.addWidget(self._build_recording_group())
+ controls_layout.addWidget(self._build_bbox_group())
- button_bar = QHBoxLayout()
+ # Preview/Stop buttons at bottom of controls - wrap in widget
+ button_bar_widget = QWidget()
+ button_bar = QHBoxLayout(button_bar_widget)
+ button_bar.setContentsMargins(0, 5, 0, 5)
self.preview_button = QPushButton("Start Preview")
+ self.preview_button.setMinimumWidth(150)
self.stop_preview_button = QPushButton("Stop Preview")
self.stop_preview_button.setEnabled(False)
+ self.stop_preview_button.setMinimumWidth(150)
button_bar.addWidget(self.preview_button)
button_bar.addWidget(self.stop_preview_button)
- controls_layout.addLayout(button_bar)
+ controls_layout.addWidget(button_bar_widget)
controls_layout.addStretch(1)
- preview_layout = QVBoxLayout()
- preview_layout.addWidget(self.video_label)
- preview_layout.addStretch(1)
-
- layout.addWidget(controls_widget)
- layout.addLayout(preview_layout, stretch=1)
+ # Add controls and video to main layout
+ layout.addWidget(controls_widget, stretch=0)
+ layout.addWidget(self.video_label, stretch=1)
self.setCentralWidget(central)
self.setStatusBar(QStatusBar())
@@ -154,7 +176,6 @@ def _build_camera_group(self) -> QGroupBox:
form = QFormLayout(group)
self.camera_backend = QComboBox()
- self.camera_backend.setEditable(True)
availability = CameraFactory.available_backends()
for backend in CameraFactory.backend_names():
label = backend
@@ -218,13 +239,6 @@ def _build_camera_group(self) -> QGroupBox:
form.addRow("Crop (x0,y0,x1,y1)", crop_layout)
- self.camera_properties_edit = QPlainTextEdit()
- self.camera_properties_edit.setPlaceholderText(
- '{"other_property": "value"}'
- )
- self.camera_properties_edit.setFixedHeight(60)
- form.addRow("Advanced properties", self.camera_properties_edit)
-
self.rotation_combo = QComboBox()
self.rotation_combo.addItem("0° (default)", 0)
self.rotation_combo.addItem("90°", 90)
@@ -245,9 +259,9 @@ def _build_dlc_group(self) -> QGroupBox:
self.model_path_edit = QLineEdit()
self.model_path_edit.setPlaceholderText("/path/to/exported/model")
path_layout.addWidget(self.model_path_edit)
- browse_model = QPushButton("Browse…")
- browse_model.clicked.connect(self._action_browse_model)
- path_layout.addWidget(browse_model)
+ self.browse_model_button = QPushButton("Browse…")
+ self.browse_model_button.clicked.connect(self._action_browse_model)
+ path_layout.addWidget(self.browse_model_button)
form.addRow("Model directory", path_layout)
self.model_type_combo = QComboBox()
@@ -256,24 +270,57 @@ def _build_dlc_group(self) -> QGroupBox:
self.model_type_combo.setCurrentIndex(0) # Default to base
form.addRow("Model type", self.model_type_combo)
+ # Processor selection
+ processor_path_layout = QHBoxLayout()
+ self.processor_folder_edit = QLineEdit()
+ self.processor_folder_edit.setText(str(Path(__file__).parent.joinpath("processors")))
+ processor_path_layout.addWidget(self.processor_folder_edit)
+
+ self.browse_processor_folder_button = QPushButton("Browse...")
+ self.browse_processor_folder_button.clicked.connect(self._action_browse_processor_folder)
+ processor_path_layout.addWidget(self.browse_processor_folder_button)
+
+ self.refresh_processors_button = QPushButton("Refresh")
+ self.refresh_processors_button.clicked.connect(self._refresh_processors)
+ processor_path_layout.addWidget(self.refresh_processors_button)
+ form.addRow("Processor folder", processor_path_layout)
+
+ self.processor_combo = QComboBox()
+ self.processor_combo.addItem("No Processor", None)
+ form.addRow("Processor", self.processor_combo)
+
self.additional_options_edit = QPlainTextEdit()
self.additional_options_edit.setPlaceholderText('')
- self.additional_options_edit.setFixedHeight(60)
+ self.additional_options_edit.setFixedHeight(40)
form.addRow("Additional options", self.additional_options_edit)
- inference_buttons = QHBoxLayout()
+ # Wrap inference buttons in a widget to prevent shifting
+ inference_button_widget = QWidget()
+ inference_buttons = QHBoxLayout(inference_button_widget)
+ inference_buttons.setContentsMargins(0, 0, 0, 0)
self.start_inference_button = QPushButton("Start pose inference")
self.start_inference_button.setEnabled(False)
+ self.start_inference_button.setMinimumWidth(150)
inference_buttons.addWidget(self.start_inference_button)
self.stop_inference_button = QPushButton("Stop pose inference")
self.stop_inference_button.setEnabled(False)
+ self.stop_inference_button.setMinimumWidth(150)
inference_buttons.addWidget(self.stop_inference_button)
- form.addRow(inference_buttons)
+ form.addRow(inference_button_widget)
self.show_predictions_checkbox = QCheckBox("Display pose predictions")
self.show_predictions_checkbox.setChecked(True)
form.addRow(self.show_predictions_checkbox)
+ self.auto_record_checkbox = QCheckBox("Auto-record video on processor command")
+ self.auto_record_checkbox.setChecked(False)
+ self.auto_record_checkbox.setToolTip("Automatically start/stop video recording when processor receives video recording commands")
+ form.addRow(self.auto_record_checkbox)
+
+ self.processor_status_label = QLabel("Processor: No clients | Recording: No")
+ self.processor_status_label.setWordWrap(True)
+ form.addRow("Processor Status", self.processor_status_label)
+
self.dlc_stats_label = QLabel("DLC processor idle")
self.dlc_stats_label.setWordWrap(True)
form.addRow("Performance", self.dlc_stats_label)
@@ -284,9 +331,6 @@ def _build_recording_group(self) -> QGroupBox:
group = QGroupBox("Recording")
form = QFormLayout(group)
- self.recording_enabled_checkbox = QCheckBox("Record video while running")
- form.addRow(self.recording_enabled_checkbox)
-
dir_layout = QHBoxLayout()
self.output_directory_edit = QLineEdit()
dir_layout.addWidget(self.output_directory_edit)
@@ -313,14 +357,18 @@ def _build_recording_group(self) -> QGroupBox:
self.crf_spin.setValue(23)
form.addRow("CRF", self.crf_spin)
+ # Wrap recording buttons in a widget to prevent shifting
+ recording_button_widget = QWidget()
+ buttons = QHBoxLayout(recording_button_widget)
+ buttons.setContentsMargins(0, 0, 0, 0)
self.start_record_button = QPushButton("Start recording")
+ self.start_record_button.setMinimumWidth(150)
+ buttons.addWidget(self.start_record_button)
self.stop_record_button = QPushButton("Stop recording")
self.stop_record_button.setEnabled(False)
-
- buttons = QHBoxLayout()
- buttons.addWidget(self.start_record_button)
+ self.stop_record_button.setMinimumWidth(150)
buttons.addWidget(self.stop_record_button)
- form.addRow(buttons)
+ form.addRow(recording_button_widget)
self.recording_stats_label = QLabel(self._last_recorder_summary)
self.recording_stats_label.setWordWrap(True)
@@ -328,6 +376,45 @@ def _build_recording_group(self) -> QGroupBox:
return group
+ def _build_bbox_group(self) -> QGroupBox:
+ """Build bounding box visualization controls."""
+ group = QGroupBox("Bounding Box Visualization")
+ form = QFormLayout(group)
+
+ self.bbox_enabled_checkbox = QCheckBox("Show bounding box")
+ self.bbox_enabled_checkbox.setChecked(False)
+ form.addRow(self.bbox_enabled_checkbox)
+
+ bbox_layout = QHBoxLayout()
+
+ self.bbox_x0_spin = QSpinBox()
+ self.bbox_x0_spin.setRange(0, 7680)
+ self.bbox_x0_spin.setPrefix("x0:")
+ self.bbox_x0_spin.setValue(0)
+ bbox_layout.addWidget(self.bbox_x0_spin)
+
+ self.bbox_y0_spin = QSpinBox()
+ self.bbox_y0_spin.setRange(0, 4320)
+ self.bbox_y0_spin.setPrefix("y0:")
+ self.bbox_y0_spin.setValue(0)
+ bbox_layout.addWidget(self.bbox_y0_spin)
+
+ self.bbox_x1_spin = QSpinBox()
+ self.bbox_x1_spin.setRange(0, 7680)
+ self.bbox_x1_spin.setPrefix("x1:")
+ self.bbox_x1_spin.setValue(100)
+ bbox_layout.addWidget(self.bbox_x1_spin)
+
+ self.bbox_y1_spin = QSpinBox()
+ self.bbox_y1_spin.setRange(0, 4320)
+ self.bbox_y1_spin.setPrefix("y1:")
+ self.bbox_y1_spin.setValue(100)
+ bbox_layout.addWidget(self.bbox_y1_spin)
+
+ form.addRow("Coordinates", bbox_layout)
+
+ return group
+
# ------------------------------------------------------------------ signals
def _connect_signals(self) -> None:
self.preview_button.clicked.connect(self._start_preview)
@@ -338,13 +425,20 @@ def _connect_signals(self) -> None:
lambda: self._refresh_camera_indices(keep_current=True)
)
self.camera_backend.currentIndexChanged.connect(self._on_backend_changed)
- self.camera_backend.editTextChanged.connect(self._on_backend_changed)
+ self.camera_backend.currentIndexChanged.connect(self._update_backend_specific_controls)
self.rotation_combo.currentIndexChanged.connect(self._on_rotation_changed)
self.start_inference_button.clicked.connect(self._start_inference)
self.stop_inference_button.clicked.connect(lambda: self._stop_inference())
self.show_predictions_checkbox.stateChanged.connect(
self._on_show_predictions_changed
)
+
+ # Connect bounding box controls
+ self.bbox_enabled_checkbox.stateChanged.connect(self._on_bbox_changed)
+ self.bbox_x0_spin.valueChanged.connect(self._on_bbox_changed)
+ self.bbox_y0_spin.valueChanged.connect(self._on_bbox_changed)
+ self.bbox_x1_spin.valueChanged.connect(self._on_bbox_changed)
+ self.bbox_y1_spin.valueChanged.connect(self._on_bbox_changed)
self.camera_controller.frame_ready.connect(self._on_frame_ready)
self.camera_controller.started.connect(self._on_camera_started)
@@ -372,22 +466,20 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.crop_y1.setValue(int(camera.crop_y1) if hasattr(camera, 'crop_y1') else 0)
backend_name = camera.backend or "opencv"
+ self.camera_backend.blockSignals(True)
index = self.camera_backend.findData(backend_name)
if index >= 0:
self.camera_backend.setCurrentIndex(index)
else:
self.camera_backend.setEditText(backend_name)
+ self.camera_backend.blockSignals(False)
self._refresh_camera_indices(keep_current=False)
self._select_camera_by_index(
camera.index, fallback_text=camera.name or str(camera.index)
)
- # Set advanced properties (exposure and gain are now separate fields)
- self.camera_properties_edit.setPlainText(
- json.dumps(camera.properties, indent=2) if camera.properties else ""
- )
-
self._active_camera_settings = None
+ self._update_backend_specific_controls()
dlc = config.dlc
self.model_path_edit.setText(dlc.model_path)
@@ -403,7 +495,6 @@ def _apply_config(self, config: ApplicationSettings) -> None:
)
recording = config.recording
- self.recording_enabled_checkbox.setChecked(recording.enabled)
self.output_directory_edit.setText(recording.directory)
self.filename_edit.setText(recording.filename)
self.container_combo.setCurrentText(recording.container)
@@ -415,11 +506,20 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.codec_combo.setCurrentIndex(self.codec_combo.count() - 1)
self.crf_spin.setValue(int(recording.crf))
+ # Set bounding box settings from config
+ bbox = config.bbox
+ self.bbox_enabled_checkbox.setChecked(bbox.enabled)
+ self.bbox_x0_spin.setValue(bbox.x0)
+ self.bbox_y0_spin.setValue(bbox.y0)
+ self.bbox_x1_spin.setValue(bbox.x1)
+ self.bbox_y1_spin.setValue(bbox.y1)
+
def _current_config(self) -> ApplicationSettings:
return ApplicationSettings(
camera=self._camera_settings_from_ui(),
dlc=self._dlc_settings_from_ui(),
recording=self._recording_settings_from_ui(),
+ bbox=self._bbox_settings_from_ui(),
)
def _camera_settings_from_ui(self) -> CameraSettings:
@@ -427,7 +527,6 @@ def _camera_settings_from_ui(self) -> CameraSettings:
if index is None:
raise ValueError("Camera selection must provide a numeric index")
backend_text = self._current_backend_name()
- properties = self._parse_json(self.camera_properties_edit.toPlainText())
# Get exposure and gain from explicit UI fields
exposure = self.camera_exposure.value()
@@ -439,12 +538,6 @@ def _camera_settings_from_ui(self) -> CameraSettings:
crop_x1 = self.crop_x1.value()
crop_y1 = self.crop_y1.value()
- # Also add to properties dict for backward compatibility with camera backends
- if exposure > 0:
- properties["exposure"] = exposure
- if gain > 0.0:
- properties["gain"] = gain
-
name_text = self.camera_index.currentText().strip()
settings = CameraSettings(
name=name_text or f"Camera {index}",
@@ -457,7 +550,7 @@ def _camera_settings_from_ui(self) -> CameraSettings:
crop_y0=crop_y0,
crop_x1=crop_x1,
crop_y1=crop_y1,
- properties=properties,
+ properties={},
)
return settings.apply_defaults()
@@ -472,7 +565,9 @@ def _refresh_camera_indices(
self, *_args: object, keep_current: bool = True
) -> None:
backend = self._current_backend_name()
- detected = CameraFactory.detect_cameras(backend)
+ # Get max_devices from config, default to 3
+ max_devices = self._config.camera.max_devices if hasattr(self._config.camera, 'max_devices') else 3
+ detected = CameraFactory.detect_cameras(backend, max_devices=max_devices)
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
logging.info(
f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
@@ -519,19 +614,6 @@ def _current_camera_index_value(self) -> Optional[int]:
return int(text)
except ValueError:
return None
-
- def _current_backend_name(self) -> str:
- backend_data = self.camera_backend.currentData()
- if isinstance(backend_data, str) and backend_data:
- return backend_data
- text = self.camera_backend.currentText().strip()
- return text or "opencv"
-
- def _refresh_camera_indices(
- self, *_args: object, keep_current: bool = True
- ) -> None:
- backend = self._current_backend_name()
- detected = CameraFactory.detect_cameras(backend)
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
logging.info(
f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
@@ -600,7 +682,7 @@ def _dlc_settings_from_ui(self) -> DLCProcessorSettings:
def _recording_settings_from_ui(self) -> RecordingSettings:
return RecordingSettings(
- enabled=self.recording_enabled_checkbox.isChecked(),
+ enabled=True, # Always enabled - recording controlled by button
directory=self.output_directory_edit.text().strip(),
filename=self.filename_edit.text().strip() or "session.mp4",
container=self.container_combo.currentText().strip() or "mp4",
@@ -608,6 +690,15 @@ def _recording_settings_from_ui(self) -> RecordingSettings:
crf=int(self.crf_spin.value()),
)
+ def _bbox_settings_from_ui(self) -> BoundingBoxSettings:
+ return BoundingBoxSettings(
+ enabled=self.bbox_enabled_checkbox.isChecked(),
+ x0=self.bbox_x0_spin.value(),
+ y0=self.bbox_y0_spin.value(),
+ x1=self.bbox_x1_spin.value(),
+ y1=self.bbox_y1_spin.value(),
+ )
+
# ------------------------------------------------------------------ actions
def _action_load_config(self) -> None:
file_name, _ = QFileDialog.getOpenFileName(
@@ -666,9 +757,66 @@ def _action_browse_directory(self) -> None:
if directory:
self.output_directory_edit.setText(directory)
+ def _action_browse_processor_folder(self) -> None:
+ """Browse for processor folder."""
+ current_path = self.processor_folder_edit.text() or "./processors"
+ directory = QFileDialog.getExistingDirectory(
+ self, "Select processor folder", current_path
+ )
+ if directory:
+ self.processor_folder_edit.setText(directory)
+ self._refresh_processors()
+
+ def _refresh_processors(self) -> None:
+ """Scan processor folder and populate dropdown."""
+ folder_path = self.processor_folder_edit.text() or "./processors"
+
+ # Clear existing items (keep "No Processor")
+ self.processor_combo.clear()
+ self.processor_combo.addItem("No Processor", None)
+
+ # Scan folder
+ try:
+ self._scanned_processors = scan_processor_folder(folder_path)
+ self._processor_keys = list(self._scanned_processors.keys())
+
+ # Populate dropdown
+ for key in self._processor_keys:
+ info = self._scanned_processors[key]
+ display_name = f"{info['name']} ({info['file']})"
+ self.processor_combo.addItem(display_name, key)
+
+ status_msg = f"Found {len(self._processor_keys)} processor(s) in {folder_path}"
+ self.statusBar().showMessage(status_msg, 3000)
+
+ except Exception as e:
+ error_msg = f"Error scanning processors: {e}"
+ self.statusBar().showMessage(error_msg, 5000)
+ logging.error(error_msg)
+ self._scanned_processors = {}
+ self._processor_keys = []
+
def _on_backend_changed(self, *_args: object) -> None:
self._refresh_camera_indices(keep_current=False)
+ def _update_backend_specific_controls(self) -> None:
+ """Enable/disable controls based on selected backend."""
+ backend = self._current_backend_name()
+ is_opencv = backend.lower() == "opencv"
+
+ # Disable exposure and gain controls for OpenCV backend
+ self.camera_exposure.setEnabled(not is_opencv)
+ self.camera_gain.setEnabled(not is_opencv)
+
+ # Set tooltip to explain why controls are disabled
+ if is_opencv:
+ tooltip = "Exposure and gain control not supported with OpenCV backend"
+ self.camera_exposure.setToolTip(tooltip)
+ self.camera_gain.setToolTip(tooltip)
+ else:
+ self.camera_exposure.setToolTip("")
+ self.camera_gain.setToolTip("")
+
def _on_rotation_changed(self, _index: int) -> None:
data = self.rotation_combo.currentData()
self._rotation_degrees = int(data) if isinstance(data, int) else 0
@@ -764,7 +912,25 @@ def _configure_dlc(self) -> bool:
if not settings.model_path:
self._show_error("Please select a DLCLive model before starting inference.")
return False
- self.dlc_processor.configure(settings)
+
+ # Instantiate processor if selected
+ processor = None
+ selected_key = self.processor_combo.currentData()
+ if selected_key is not None and self._scanned_processors:
+ try:
+ # For now, instantiate with no parameters
+ # TODO: Add parameter dialog for processors that need params
+ # or pass kwargs from config ?
+ processor = instantiate_from_scan(self._scanned_processors, selected_key)
+ processor_name = self._scanned_processors[selected_key]['name']
+ self.statusBar().showMessage(f"Loaded processor: {processor_name}", 3000)
+ except Exception as e:
+ error_msg = f"Failed to instantiate processor: {e}"
+ self._show_error(error_msg)
+ logging.error(error_msg)
+ return False
+
+ self.dlc_processor.configure(settings, processor=processor)
return True
def _update_inference_buttons(self) -> None:
@@ -772,6 +938,22 @@ def _update_inference_buttons(self) -> None:
self.start_inference_button.setEnabled(preview_running and not self._dlc_active)
self.stop_inference_button.setEnabled(preview_running and self._dlc_active)
+ def _update_dlc_controls_enabled(self) -> None:
+ """Enable/disable DLC settings based on inference state."""
+ allow_changes = not self._dlc_active
+ widgets = [
+ self.model_path_edit,
+ self.browse_model_button,
+ self.model_type_combo,
+ self.processor_folder_edit,
+ self.browse_processor_folder_button,
+ self.refresh_processors_button,
+ self.processor_combo,
+ self.additional_options_edit,
+ ]
+ for widget in widgets:
+ widget.setEnabled(allow_changes)
+
def _update_camera_controls_enabled(self) -> None:
recording_active = (
self._video_recorder is not None and self._video_recorder.is_running
@@ -792,7 +974,6 @@ def _update_camera_controls_enabled(self) -> None:
self.crop_y0,
self.crop_x1,
self.crop_y1,
- self.camera_properties_edit,
self.rotation_combo,
self.codec_combo,
self.crf_spin,
@@ -871,6 +1052,10 @@ def _update_metrics(self) -> None:
else:
self.dlc_stats_label.setText("DLC processor idle")
+ # Update processor status (connection and recording state)
+ if hasattr(self, "processor_status_label"):
+ self._update_processor_status()
+
if hasattr(self, "recording_stats_label"):
if self._video_recorder is not None:
stats = self._video_recorder.get_stats()
@@ -884,6 +1069,62 @@ def _update_metrics(self) -> None:
else:
self.recording_stats_label.setText(self._last_recorder_summary)
+ def _update_processor_status(self) -> None:
+ """Update processor connection and recording status, handle auto-recording."""
+ if not self._dlc_active or not self._dlc_initialized:
+ self.processor_status_label.setText("Processor: Not active")
+ return
+
+ # Get processor instance from dlc_processor
+ processor = self.dlc_processor._processor
+
+ if processor is None:
+ self.processor_status_label.setText("Processor: None loaded")
+ return
+
+ # Check if processor has the required attributes (socket-based processors)
+ if not hasattr(processor, 'conns') or not hasattr(processor, '_recording'):
+ self.processor_status_label.setText("Processor: No status info")
+ return
+
+ # Get connection count and recording state
+ num_clients = len(processor.conns)
+ is_recording = processor.recording if hasattr(processor, 'recording') else False
+
+ # Format status message
+ client_str = f"{num_clients} client{'s' if num_clients != 1 else ''}"
+ recording_str = "Yes" if is_recording else "No"
+ self.processor_status_label.setText(f"Clients: {client_str} | Recording: {recording_str}")
+
+ # Handle auto-recording based on processor's video recording flag
+ if hasattr(processor, '_vid_recording') and self.auto_record_checkbox.isChecked():
+ current_vid_recording = processor.video_recording
+
+ # Check if video recording state changed
+ if current_vid_recording != self._last_processor_vid_recording:
+ if current_vid_recording:
+ # Start video recording
+ if not self._video_recorder or not self._video_recorder.is_running:
+ # Get session name from processor
+ session_name = getattr(processor, 'session_name', 'auto_session')
+ self._auto_record_session_name = session_name
+
+ # Update filename with session name
+ original_filename = self.filename_edit.text()
+ self.filename_edit.setText(f"{session_name}.mp4")
+
+ self._start_recording()
+ self.statusBar().showMessage(f"Auto-started recording: {session_name}", 3000)
+ logging.info(f"Auto-recording started for session: {session_name}")
+ else:
+ # Stop video recording
+ if self._video_recorder and self._video_recorder.is_running:
+ self._stop_recording()
+ self.statusBar().showMessage("Auto-stopped recording", 3000)
+ logging.info("Auto-recording stopped")
+
+ self._last_processor_vid_recording = current_vid_recording
+
def _start_inference(self) -> None:
if self._dlc_active:
self.statusBar().showMessage("Pose inference already running", 3000)
@@ -909,6 +1150,7 @@ def _start_inference(self) -> None:
self.statusBar().showMessage("Initializing DLCLive…", 3000)
self._update_camera_controls_enabled()
+ self._update_dlc_controls_enabled()
def _stop_inference(self, show_message: bool = True) -> None:
was_active = self._dlc_active
@@ -916,6 +1158,8 @@ def _stop_inference(self, show_message: bool = True) -> None:
self._dlc_initialized = False
self.dlc_processor.reset()
self._last_pose = None
+ self._last_processor_vid_recording = False
+ self._auto_record_session_name = None
# Reset button appearance
self.start_inference_button.setText("Start pose inference")
@@ -927,6 +1171,7 @@ def _stop_inference(self, show_message: bool = True) -> None:
self.statusBar().showMessage("Pose inference stopped", 3000)
self._update_inference_buttons()
self._update_camera_controls_enabled()
+ self._update_dlc_controls_enabled()
# ------------------------------------------------------------------ recording
def _start_recording(self) -> None:
@@ -1026,9 +1271,10 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
# Check if it's a frame size error
if "Frame size changed" in str(exc):
self._show_warning(f"Camera resolution changed - restarting recording: {exc}")
+ was_recording = self._video_recorder and self._video_recorder.is_running
self._stop_recording()
- # Restart recording with new resolution if enabled
- if self.recording_enabled_checkbox.isChecked():
+ # Restart recording with new resolution if it was already running
+ if was_recording:
try:
self._start_recording()
except Exception as restart_exc:
@@ -1060,6 +1306,11 @@ def _update_video_display(self, frame: np.ndarray) -> None:
and self._last_pose.pose is not None
):
display_frame = self._draw_pose(frame, self._last_pose.pose)
+
+ # Draw bounding box if enabled
+ if self._bbox_enabled:
+ display_frame = self._draw_bbox(display_frame)
+
rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB)
h, w, ch = rgb.shape
bytes_per_line = ch * w
@@ -1104,6 +1355,41 @@ def _on_show_predictions_changed(self, _state: int) -> None:
if self._current_frame is not None:
self._display_frame(self._current_frame, force=True)
+ def _on_bbox_changed(self, _value: int = 0) -> None:
+ """Handle bounding box parameter changes."""
+ self._bbox_enabled = self.bbox_enabled_checkbox.isChecked()
+ self._bbox_x0 = self.bbox_x0_spin.value()
+ self._bbox_y0 = self.bbox_y0_spin.value()
+ self._bbox_x1 = self.bbox_x1_spin.value()
+ self._bbox_y1 = self.bbox_y1_spin.value()
+
+ # Force redraw if preview is running
+ if self._current_frame is not None:
+ self._display_frame(self._current_frame, force=True)
+
+ def _draw_bbox(self, frame: np.ndarray) -> np.ndarray:
+ """Draw bounding box on frame with red lines."""
+ overlay = frame.copy()
+ x0 = self._bbox_x0
+ y0 = self._bbox_y0
+ x1 = self._bbox_x1
+ y1 = self._bbox_y1
+
+ # Validate coordinates
+ if x0 >= x1 or y0 >= y1:
+ return overlay
+
+ height, width = frame.shape[:2]
+ x0 = max(0, min(x0, width - 1))
+ y0 = max(0, min(y0, height - 1))
+ x1 = max(x0 + 1, min(x1, width))
+ y1 = max(y0 + 1, min(y1, height))
+
+ # Draw red rectangle (BGR format: red is (0, 0, 255))
+ cv2.rectangle(overlay, (x0, y0), (x1, y1), (0, 0, 255), 2)
+
+ return overlay
+
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
for keypoint in np.asarray(pose):
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index dbb0f90..3f5b951 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -134,6 +134,7 @@ def __init__(
self._session_name = "test_session"
self.filename = None
self._recording = Event() # Thread-safe recording flag
+ self._vid_recording = Event() # Thread-safe video recording flag
# State
self.curr_step = 0
@@ -144,6 +145,11 @@ def recording(self):
"""Thread-safe recording flag."""
return self._recording.is_set()
+ @property
+ def video_recording(self):
+ """Thread-safe video recording flag."""
+ return self._vid_recording.is_set()
+
@property
def session_name(self):
return self._session_name
@@ -155,11 +161,11 @@ def session_name(self, name):
def _accept_loop(self):
"""Background thread to accept new client connections."""
- LOG.info(f"DLC Processor listening on {self.address[0]}:{self.address[1]}")
+ LOG.debug(f"DLC Processor listening on {self.address[0]}:{self.address[1]}")
while not self._stop.is_set():
try:
c = self.listener.accept()
- LOG.info(f"Client connected from {self.listener.last_accepted}")
+ LOG.debug(f"Client connected from {self.listener.last_accepted}")
self.conns.add(c)
# Start RX loop for this connection (in case clients send data)
Thread(target=self._rx_loop, args=(c,), name="DLCRX", daemon=True).start()
@@ -195,6 +201,7 @@ def _handle_client_message(self, msg):
LOG.info(f"Session name set to: {session_name}")
elif cmd == "start_recording":
+ self._vid_recording.set()
self._recording.set()
# Clear all data queues
self._clear_data_queues()
@@ -203,12 +210,18 @@ def _handle_client_message(self, msg):
elif cmd == "stop_recording":
self._recording.clear()
+ self._vid_recording.clear()
LOG.info("Recording stopped")
elif cmd == "save":
filename = msg.get("filename", self.filename)
save_code = self.save(filename)
LOG.info(f"Save {'successful' if save_code == 1 else 'failed'}: {filename}")
+
+ elif cmd == "start_video":
+ # Placeholder for video recording start
+ self._vid_recording.set()
+ LOG.info("Start video recording command received")
def _clear_data_queues(self):
"""Clear all data storage queues. Override in subclasses to clear additional queues."""
diff --git a/example_recording_control.py b/example_recording_control.py
deleted file mode 100644
index ed426e8..0000000
--- a/example_recording_control.py
+++ /dev/null
@@ -1,194 +0,0 @@
-"""
-Example: Recording control with DLCClient and MyProcessor_socket
-
-This demonstrates:
-1. Starting a processor
-2. Connecting a client
-3. Controlling recording (start/stop/save) from the client
-4. Session name management
-"""
-
-import time
-import sys
-from pathlib import Path
-
-# Add parent directory to path
-sys.path.insert(0, str(Path(__file__).parent.parent.parent))
-
-from mouse_ar.ctrl.dlc_client import DLCClient
-
-
-def example_recording_workflow():
- """Complete workflow: processor + client with recording control."""
-
- print("\n" + "="*70)
- print("EXAMPLE: Recording Control Workflow")
- print("="*70)
-
- # NOTE: This example assumes MyProcessor_socket is already running
- # Start it separately with:
- # from dlc_processor_socket import MyProcessor_socket
- # processor = MyProcessor_socket(bind=("localhost", 6000))
- # # Then run DLCLive with this processor
-
- print("\n[CLIENT] Connecting to processor at localhost:6000...")
- client = DLCClient(address=("localhost", 6000))
-
- try:
- # Start the client (connects and begins receiving data)
- client.start()
- print("[CLIENT] Connected!")
- time.sleep(0.5) # Wait for connection to stabilize
-
- # Set session name
- print("\n[CLIENT] Setting session name to 'experiment_001'...")
- client.set_session_name("experiment_001")
- time.sleep(0.2)
-
- # Start recording
- print("[CLIENT] Starting recording (clears processor data queues)...")
- client.start_recording()
- time.sleep(0.2)
-
- # Receive some data
- print("\n[CLIENT] Receiving data for 5 seconds...")
- for i in range(5):
- data = client.read()
- if data:
- vals = data["vals"]
- print(f" t={vals[0]:.2f}, x={vals[1]:.1f}, y={vals[2]:.1f}, "
- f"heading={vals[3]:.1f}°, head_angle={vals[4]:.2f}rad")
- time.sleep(1.0)
-
- # Stop recording
- print("\n[CLIENT] Stopping recording...")
- client.stop_recording()
- time.sleep(0.2)
-
- # Trigger save
- print("[CLIENT] Triggering save on processor...")
- client.trigger_save() # Uses processor's default filename
- # OR specify custom filename:
- # client.trigger_save(filename="my_custom_data.pkl")
- time.sleep(0.5)
-
- print("\n[CLIENT] ✓ Workflow complete!")
-
- except Exception as e:
- print(f"\n[ERROR] {e}")
- print("\nMake sure MyProcessor_socket is running!")
- print("Example:")
- print(" from dlc_processor_socket import MyProcessor_socket")
- print(" processor = MyProcessor_socket()")
- print(" # Then run DLCLive with this processor")
-
- finally:
- print("\n[CLIENT] Closing connection...")
- client.close()
-
-
-def example_multiple_sessions():
- """Example: Recording multiple sessions with the same processor."""
-
- print("\n" + "="*70)
- print("EXAMPLE: Multiple Sessions")
- print("="*70)
-
- client = DLCClient(address=("localhost", 6000))
-
- try:
- client.start()
- print("[CLIENT] Connected!")
- time.sleep(0.5)
-
- # Session 1
- print("\n--- SESSION 1 ---")
- client.set_session_name("trial_001")
- client.start_recording()
- print("Recording session 'trial_001' for 3 seconds...")
- time.sleep(3.0)
- client.stop_recording()
- client.trigger_save() # Saves as "trial_001_dlc_processor_data.pkl"
- print("Session 1 saved!")
-
- time.sleep(1.0)
-
- # Session 2
- print("\n--- SESSION 2 ---")
- client.set_session_name("trial_002")
- client.start_recording()
- print("Recording session 'trial_002' for 3 seconds...")
- time.sleep(3.0)
- client.stop_recording()
- client.trigger_save() # Saves as "trial_002_dlc_processor_data.pkl"
- print("Session 2 saved!")
-
- print("\n✓ Multiple sessions recorded successfully!")
-
- except Exception as e:
- print(f"\n[ERROR] {e}")
-
- finally:
- client.close()
-
-
-def example_command_api():
- """Example: Using the low-level command API."""
-
- print("\n" + "="*70)
- print("EXAMPLE: Low-level Command API")
- print("="*70)
-
- client = DLCClient(address=("localhost", 6000))
-
- try:
- client.start()
- time.sleep(0.5)
-
- # Using send_command directly
- print("\n[CLIENT] Using send_command()...")
-
- # Set session name
- client.send_command("set_session_name", session_name="custom_session")
- print(" ✓ Sent: set_session_name")
- time.sleep(0.2)
-
- # Start recording
- client.send_command("start_recording")
- print(" ✓ Sent: start_recording")
- time.sleep(2.0)
-
- # Stop recording
- client.send_command("stop_recording")
- print(" ✓ Sent: stop_recording")
- time.sleep(0.2)
-
- # Save with custom filename
- client.send_command("save", filename="my_data.pkl")
- print(" ✓ Sent: save")
- time.sleep(0.5)
-
- print("\n✓ Commands sent successfully!")
-
- except Exception as e:
- print(f"\n[ERROR] {e}")
-
- finally:
- client.close()
-
-
-if __name__ == "__main__":
- print("\n" + "="*70)
- print("DLC PROCESSOR RECORDING CONTROL EXAMPLES")
- print("="*70)
- print("\nNOTE: These examples require MyProcessor_socket to be running.")
- print("Start it separately before running these examples.")
- print("="*70)
-
- # Uncomment the example you want to run:
-
- # example_recording_workflow()
- # example_multiple_sessions()
- # example_command_api()
-
- print("\nUncomment an example in the script to run it.")
From f1ab8b02c28e3553097598fe451cae37bb58f901 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 24 Oct 2025 15:12:44 +0200
Subject: [PATCH 13/26] formatting and precommit workflow
---
.github/workflows/format.yml | 23 ++
.gitignore | 1 -
.pre-commit-config.yaml | 23 ++
dlclivegui/__init__.py | 8 +-
dlclivegui/camera_controller.py | 99 ++++---
dlclivegui/cameras/__init__.py | 1 +
dlclivegui/cameras/base.py | 1 +
dlclivegui/cameras/basler_backend.py | 9 +-
dlclivegui/cameras/factory.py | 3 +-
dlclivegui/cameras/gentl_backend.py | 46 ++--
dlclivegui/cameras/opencv_backend.py | 11 +-
dlclivegui/config.py | 13 +-
dlclivegui/dlc_processor.py | 62 +++--
dlclivegui/gui.py | 243 ++++++++----------
dlclivegui/processors/GUI_INTEGRATION.md | 12 +-
dlclivegui/processors/PLUGIN_SYSTEM.md | 4 +-
dlclivegui/processors/dlc_processor_socket.py | 62 ++---
dlclivegui/processors/processor_utils.py | 43 ++--
dlclivegui/video_recorder.py | 41 ++-
setup.py | 1 +
20 files changed, 378 insertions(+), 328 deletions(-)
create mode 100644 .github/workflows/format.yml
create mode 100644 .pre-commit-config.yaml
diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
new file mode 100644
index 0000000..a226ae0
--- /dev/null
+++ b/.github/workflows/format.yml
@@ -0,0 +1,23 @@
+name: pre-commit-format
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+ branches: [main]
+
+jobs:
+ pre_commit_checks:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ ref: ${{ github.head_ref }}
+
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '3.10'
+
+ - run: pip install pre-commit
+ - run: pre-commit run --all-files
diff --git a/.gitignore b/.gitignore
index 1a13ced..d2ee717 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,7 +2,6 @@
### DLC Live Specific
#####################
-*config*
**test*
###################
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..09dc3cd
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,23 @@
+repos:
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.4.0
+ hooks:
+ - id: check-added-large-files
+ - id: check-yaml
+ - id: end-of-file-fixer
+ - id: name-tests-test
+ args: [--pytest-test-first]
+ - id: trailing-whitespace
+ - id: check-merge-conflict
+
+ - repo: https://github.com/PyCQA/isort
+ rev: 5.12.0
+ hooks:
+ - id: isort
+ args: ["--profile", "black", "--line-length", "100", "--atomic"]
+
+ - repo: https://github.com/psf/black
+ rev: 24.4.2
+ hooks:
+ - id: black
+ args: ["--line-length=100"]
diff --git a/dlclivegui/__init__.py b/dlclivegui/__init__.py
index 1408486..d91f23b 100644
--- a/dlclivegui/__init__.py
+++ b/dlclivegui/__init__.py
@@ -1,10 +1,6 @@
"""DeepLabCut Live GUI package."""
-from .config import (
- ApplicationSettings,
- CameraSettings,
- DLCProcessorSettings,
- RecordingSettings,
-)
+
+from .config import ApplicationSettings, CameraSettings, DLCProcessorSettings, RecordingSettings
from .gui import MainWindow, main
__all__ = [
diff --git a/dlclivegui/camera_controller.py b/dlclivegui/camera_controller.py
index 5618163..7c80df1 100644
--- a/dlclivegui/camera_controller.py
+++ b/dlclivegui/camera_controller.py
@@ -1,4 +1,5 @@
"""Camera management for the DLC Live GUI."""
+
from __future__ import annotations
import logging
@@ -8,7 +9,7 @@
from typing import Optional
import numpy as np
-from PyQt6.QtCore import QObject, QThread, QMetaObject, Qt, pyqtSignal, pyqtSlot
+from PyQt6.QtCore import QMetaObject, QObject, Qt, QThread, pyqtSignal, pyqtSlot
from dlclivegui.cameras import CameraFactory
from dlclivegui.cameras.base import CameraBackend
@@ -39,20 +40,20 @@ def __init__(self, settings: CameraSettings):
self._settings = settings
self._stop_event = Event()
self._backend: Optional[CameraBackend] = None
-
+
# Error recovery settings
self._max_consecutive_errors = 5
self._max_reconnect_attempts = 3
self._retry_delay = 0.1 # seconds
self._reconnect_delay = 1.0 # seconds
-
+
# Frame validation
self._expected_frame_size: Optional[tuple[int, int]] = None # (height, width)
@pyqtSlot()
def run(self) -> None:
self._stop_event.clear()
-
+
# Initialize camera
if not self._initialize_camera():
self.finished.emit()
@@ -66,30 +67,36 @@ def run(self) -> None:
while not self._stop_event.is_set():
try:
frame, timestamp = self._backend.read()
-
+
# Validate frame size
if not self._validate_frame_size(frame):
consecutive_errors += 1
- LOGGER.warning(f"Frame size validation failed ({consecutive_errors}/{self._max_consecutive_errors})")
+ LOGGER.warning(
+ f"Frame size validation failed ({consecutive_errors}/{self._max_consecutive_errors})"
+ )
if consecutive_errors >= self._max_consecutive_errors:
self.error_occurred.emit("Too many frames with incorrect size")
break
time.sleep(self._retry_delay)
continue
-
+
consecutive_errors = 0 # Reset error count on success
reconnect_attempts = 0 # Reset reconnect attempts on success
-
+
except TimeoutError as exc:
consecutive_errors += 1
- LOGGER.warning(f"Camera frame timeout ({consecutive_errors}/{self._max_consecutive_errors}): {exc}")
-
+ LOGGER.warning(
+ f"Camera frame timeout ({consecutive_errors}/{self._max_consecutive_errors}): {exc}"
+ )
+
if self._stop_event.is_set():
break
-
+
# Handle timeout with retry logic
if consecutive_errors < self._max_consecutive_errors:
- self.warning_occurred.emit(f"Frame timeout (retry {consecutive_errors}/{self._max_consecutive_errors})")
+ self.warning_occurred.emit(
+ f"Frame timeout (retry {consecutive_errors}/{self._max_consecutive_errors})"
+ )
time.sleep(self._retry_delay)
continue
else:
@@ -98,29 +105,39 @@ def run(self) -> None:
if self._attempt_reconnection():
consecutive_errors = 0
reconnect_attempts += 1
- self.warning_occurred.emit(f"Camera reconnected (attempt {reconnect_attempts})")
+ self.warning_occurred.emit(
+ f"Camera reconnected (attempt {reconnect_attempts})"
+ )
continue
else:
reconnect_attempts += 1
if reconnect_attempts >= self._max_reconnect_attempts:
- self.error_occurred.emit(f"Camera reconnection failed after {reconnect_attempts} attempts")
+ self.error_occurred.emit(
+ f"Camera reconnection failed after {reconnect_attempts} attempts"
+ )
break
else:
consecutive_errors = 0 # Reset to try again
- self.warning_occurred.emit(f"Reconnection attempt {reconnect_attempts} failed, retrying...")
+ self.warning_occurred.emit(
+ f"Reconnection attempt {reconnect_attempts} failed, retrying..."
+ )
time.sleep(self._reconnect_delay)
continue
-
+
except Exception as exc:
consecutive_errors += 1
- LOGGER.warning(f"Camera read error ({consecutive_errors}/{self._max_consecutive_errors}): {exc}")
-
+ LOGGER.warning(
+ f"Camera read error ({consecutive_errors}/{self._max_consecutive_errors}): {exc}"
+ )
+
if self._stop_event.is_set():
break
-
+
# Handle general errors with retry logic
if consecutive_errors < self._max_consecutive_errors:
- self.warning_occurred.emit(f"Frame read error (retry {consecutive_errors}/{self._max_consecutive_errors})")
+ self.warning_occurred.emit(
+ f"Frame read error (retry {consecutive_errors}/{self._max_consecutive_errors})"
+ )
time.sleep(self._retry_delay)
continue
else:
@@ -129,22 +146,28 @@ def run(self) -> None:
if self._attempt_reconnection():
consecutive_errors = 0
reconnect_attempts += 1
- self.warning_occurred.emit(f"Camera reconnected (attempt {reconnect_attempts})")
+ self.warning_occurred.emit(
+ f"Camera reconnected (attempt {reconnect_attempts})"
+ )
continue
else:
reconnect_attempts += 1
if reconnect_attempts >= self._max_reconnect_attempts:
- self.error_occurred.emit(f"Camera failed after {reconnect_attempts} reconnection attempts: {exc}")
+ self.error_occurred.emit(
+ f"Camera failed after {reconnect_attempts} reconnection attempts: {exc}"
+ )
break
else:
consecutive_errors = 0 # Reset to try again
- self.warning_occurred.emit(f"Reconnection attempt {reconnect_attempts} failed, retrying...")
+ self.warning_occurred.emit(
+ f"Reconnection attempt {reconnect_attempts} failed, retrying..."
+ )
time.sleep(self._reconnect_delay)
continue
-
+
if self._stop_event.is_set():
break
-
+
self.frame_captured.emit(FrameData(frame, timestamp))
# Cleanup
@@ -158,7 +181,9 @@ def _initialize_camera(self) -> bool:
self._backend.open()
# Don't set expected frame size - will be established from first frame
self._expected_frame_size = None
- LOGGER.info("Camera initialized successfully, frame size will be determined from camera")
+ LOGGER.info(
+ "Camera initialized successfully, frame size will be determined from camera"
+ )
return True
except Exception as exc:
LOGGER.exception("Failed to initialize camera", exc_info=exc)
@@ -170,15 +195,17 @@ def _validate_frame_size(self, frame: np.ndarray) -> bool:
if frame is None or frame.size == 0:
LOGGER.warning("Received empty frame")
return False
-
+
actual_size = (frame.shape[0], frame.shape[1]) # (height, width)
-
+
if self._expected_frame_size is None:
# First frame - establish expected size
self._expected_frame_size = actual_size
- LOGGER.info(f"Established expected frame size: (h={actual_size[0]}, w={actual_size[1]})")
+ LOGGER.info(
+ f"Established expected frame size: (h={actual_size[0]}, w={actual_size[1]})"
+ )
return True
-
+
if actual_size != self._expected_frame_size:
LOGGER.warning(
f"Frame size mismatch: expected (h={self._expected_frame_size[0]}, w={self._expected_frame_size[1]}), "
@@ -192,26 +219,26 @@ def _validate_frame_size(self, frame: np.ndarray) -> bool:
f"Camera resolution changed to {actual_size[1]}x{actual_size[0]}"
)
return True # Accept the new size
-
+
return True
def _attempt_reconnection(self) -> bool:
"""Attempt to reconnect to the camera. Returns True on success, False on failure."""
if self._stop_event.is_set():
return False
-
+
LOGGER.info("Attempting camera reconnection...")
-
+
# Close existing connection
self._cleanup_camera()
-
+
# Wait longer before reconnecting to let the device fully release
LOGGER.info(f"Waiting {self._reconnect_delay}s before reconnecting...")
time.sleep(self._reconnect_delay)
-
+
if self._stop_event.is_set():
return False
-
+
# Try to reinitialize (this will also reset expected frame size)
try:
self._backend = CameraFactory.create(self._settings)
diff --git a/dlclivegui/cameras/__init__.py b/dlclivegui/cameras/__init__.py
index cf7f488..7aa4621 100644
--- a/dlclivegui/cameras/__init__.py
+++ b/dlclivegui/cameras/__init__.py
@@ -1,4 +1,5 @@
"""Camera backend implementations and factory helpers."""
+
from __future__ import annotations
from .factory import CameraFactory
diff --git a/dlclivegui/cameras/base.py b/dlclivegui/cameras/base.py
index 910331c..f060d8b 100644
--- a/dlclivegui/cameras/base.py
+++ b/dlclivegui/cameras/base.py
@@ -1,4 +1,5 @@
"""Abstract camera backend definitions."""
+
from __future__ import annotations
from abc import ABC, abstractmethod
diff --git a/dlclivegui/cameras/basler_backend.py b/dlclivegui/cameras/basler_backend.py
index e83185a..ec23806 100644
--- a/dlclivegui/cameras/basler_backend.py
+++ b/dlclivegui/cameras/basler_backend.py
@@ -1,4 +1,5 @@
"""Basler camera backend implemented with :mod:`pypylon`."""
+
from __future__ import annotations
import time
@@ -28,9 +29,7 @@ def is_available(cls) -> bool:
def open(self) -> None:
if pylon is None: # pragma: no cover - optional dependency
- raise RuntimeError(
- "pypylon is required for the Basler backend but is not installed"
- )
+ raise RuntimeError("pypylon is required for the Basler backend but is not installed")
devices = self._enumerate_devices()
if not devices:
raise RuntimeError("No Basler cameras detected")
@@ -114,7 +113,9 @@ def _enumerate_devices(self):
return factory.EnumerateDevices()
def _select_device(self, devices):
- serial = self.settings.properties.get("serial") or self.settings.properties.get("serial_number")
+ serial = self.settings.properties.get("serial") or self.settings.properties.get(
+ "serial_number"
+ )
if serial:
for device in devices:
if getattr(device, "GetSerialNumber", None) and device.GetSerialNumber() == serial:
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index 540e352..eca4f58 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -1,4 +1,5 @@
"""Backend discovery and construction utilities."""
+
from __future__ import annotations
import importlib
@@ -74,7 +75,7 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
# For GenTL backend, try to get actual device count
num_devices = max_devices
- if hasattr(backend_cls, 'get_device_count'):
+ if hasattr(backend_cls, "get_device_count"):
try:
actual_count = backend_cls.get_device_count()
if actual_count >= 0:
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 943cf4a..701d4fd 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -1,4 +1,5 @@
"""GenTL backend implemented using the Harvesters library."""
+
from __future__ import annotations
import glob
@@ -13,6 +14,7 @@
try: # pragma: no cover - optional dependency
from harvesters.core import Harvester
+
try:
from harvesters.core import HarvesterTimeoutError # type: ignore
except Exception: # pragma: no cover - optional dependency
@@ -43,7 +45,9 @@ def __init__(self, settings):
self._exposure: Optional[float] = props.get("exposure")
self._gain: Optional[float] = props.get("gain")
self._timeout: float = float(props.get("timeout", 2.0))
- self._cti_search_paths: Tuple[str, ...] = self._parse_cti_paths(props.get("cti_search_paths"))
+ self._cti_search_paths: Tuple[str, ...] = self._parse_cti_paths(
+ props.get("cti_search_paths")
+ )
self._harvester = None
self._acquirer = None
@@ -56,21 +60,21 @@ def is_available(cls) -> bool:
@classmethod
def get_device_count(cls) -> int:
"""Get the actual number of GenTL devices detected by Harvester.
-
+
Returns the number of devices found, or -1 if detection fails.
"""
if Harvester is None:
return -1
-
+
harvester = None
try:
harvester = Harvester()
# Use the static helper to find CTI file with default patterns
cti_file = cls._search_cti_file(cls._DEFAULT_CTI_PATTERNS)
-
+
if not cti_file:
return -1
-
+
harvester.add_file(cti_file)
harvester.update()
return len(harvester.device_info_list)
@@ -120,28 +124,28 @@ def open(self) -> None:
remote = self._acquirer.remote_device
node_map = remote.node_map
- #print(dir(node_map))
+ # print(dir(node_map))
"""
- ['AcquisitionBurstFrameCount', 'AcquisitionControl', 'AcquisitionFrameRate', 'AcquisitionMode',
+ ['AcquisitionBurstFrameCount', 'AcquisitionControl', 'AcquisitionFrameRate', 'AcquisitionMode',
'AcquisitionStart', 'AcquisitionStop', 'AnalogControl', 'AutoFunctionsROI', 'AutoFunctionsROIEnable',
'AutoFunctionsROIHeight', 'AutoFunctionsROILeft', 'AutoFunctionsROIPreset', 'AutoFunctionsROITop',
'AutoFunctionsROIWidth', 'BinningHorizontal', 'BinningVertical', 'BlackLevel', 'CameraRegisterAddress',
- 'CameraRegisterAddressSpace', 'CameraRegisterControl', 'CameraRegisterRead', 'CameraRegisterValue',
+ 'CameraRegisterAddressSpace', 'CameraRegisterControl', 'CameraRegisterRead', 'CameraRegisterValue',
'CameraRegisterWrite', 'Contrast', 'DecimationHorizontal', 'DecimationVertical', 'Denoise',
- 'DeviceControl', 'DeviceFirmwareVersion', 'DeviceModelName', 'DeviceReset', 'DeviceSFNCVersionMajor',
- 'DeviceSFNCVersionMinor', 'DeviceSFNCVersionSubMinor', 'DeviceScanType', 'DeviceSerialNumber',
- 'DeviceTLType', 'DeviceTLVersionMajor', 'DeviceTLVersionMinor', 'DeviceTLVersionSubMinor',
- 'DeviceTemperature', 'DeviceTemperatureSelector', 'DeviceType', 'DeviceUserID', 'DeviceVendorName',
- 'DigitalIO', 'ExposureAuto', 'ExposureAutoHighlightReduction', 'ExposureAutoLowerLimit',
- 'ExposureAutoReference', 'ExposureAutoUpperLimit', 'ExposureAutoUpperLimitAuto', 'ExposureTime',
+ 'DeviceControl', 'DeviceFirmwareVersion', 'DeviceModelName', 'DeviceReset', 'DeviceSFNCVersionMajor',
+ 'DeviceSFNCVersionMinor', 'DeviceSFNCVersionSubMinor', 'DeviceScanType', 'DeviceSerialNumber',
+ 'DeviceTLType', 'DeviceTLVersionMajor', 'DeviceTLVersionMinor', 'DeviceTLVersionSubMinor',
+ 'DeviceTemperature', 'DeviceTemperatureSelector', 'DeviceType', 'DeviceUserID', 'DeviceVendorName',
+ 'DigitalIO', 'ExposureAuto', 'ExposureAutoHighlightReduction', 'ExposureAutoLowerLimit',
+ 'ExposureAutoReference', 'ExposureAutoUpperLimit', 'ExposureAutoUpperLimitAuto', 'ExposureTime',
'GPIn', 'GPOut', 'Gain', 'GainAuto', 'GainAutoLowerLimit', 'GainAutoUpperLimit', 'Gamma', 'Height',
'HeightMax', 'IMXLowLatencyTriggerMode', 'ImageFormatControl', 'OffsetAutoCenter', 'OffsetX', 'OffsetY',
'PayloadSize', 'PixelFormat', 'ReverseX', 'ReverseY', 'Root', 'SensorHeight', 'SensorWidth', 'Sharpness',
'ShowOverlay', 'SoftwareAnalogControl', 'SoftwareTransformControl', 'SoftwareTransformEnable',
'StrobeDelay', 'StrobeDuration', 'StrobeEnable', 'StrobeOperation', 'StrobePolarity', 'TLParamsLocked',
- 'TestControl', 'TestPendingAck', 'TimestampLatch', 'TimestampLatchValue', 'TimestampReset', 'ToneMappingAuto',
- 'ToneMappingControl', 'ToneMappingEnable', 'ToneMappingGlobalBrightness', 'ToneMappingIntensity',
- 'TransportLayerControl', 'TriggerActivation', 'TriggerDebouncer', 'TriggerDelay', 'TriggerDenoise',
+ 'TestControl', 'TestPendingAck', 'TimestampLatch', 'TimestampLatchValue', 'TimestampReset', 'ToneMappingAuto',
+ 'ToneMappingControl', 'ToneMappingEnable', 'ToneMappingGlobalBrightness', 'ToneMappingIntensity',
+ 'TransportLayerControl', 'TriggerActivation', 'TriggerDebouncer', 'TriggerDelay', 'TriggerDenoise',
'TriggerMask', 'TriggerMode', 'TriggerOverlap', 'TriggerSelector', 'TriggerSoftware', 'TriggerSource',
'UserSetControl', 'UserSetDefault', 'UserSetLoad', 'UserSetSave', 'UserSetSelector', 'Width', 'WidthMax']
"""
@@ -176,7 +180,7 @@ def read(self) -> Tuple[np.ndarray, float]:
except ValueError:
frame = array.copy()
except HarvesterTimeoutError as exc:
- raise TimeoutError(str(exc)+ " (GenTL timeout)") from exc
+ raise TimeoutError(str(exc) + " (GenTL timeout)") from exc
frame = self._convert_frame(frame)
timestamp = time.time()
@@ -231,7 +235,7 @@ def _parse_crop(self, crop) -> Optional[Tuple[int, int, int, int]]:
@staticmethod
def _search_cti_file(patterns: Tuple[str, ...]) -> Optional[str]:
"""Search for a CTI file using the given patterns.
-
+
Returns the first CTI file found, or None if none found.
"""
for pattern in patterns:
@@ -242,7 +246,7 @@ def _search_cti_file(patterns: Tuple[str, ...]) -> Optional[str]:
def _find_cti_file(self) -> str:
"""Find a CTI file using configured or default search paths.
-
+
Raises RuntimeError if no CTI file is found.
"""
cti_file = self._search_cti_file(self._cti_search_paths)
@@ -266,7 +270,7 @@ def _create_acquirer(self, serial: Optional[str], index: int):
assert self._harvester is not None
methods = [
getattr(self._harvester, "create", None),
- getattr(self._harvester, "create_image_acquirer", None),
+ getattr(self._harvester, "create_image_acquirer", None),
]
methods = [m for m in methods if m is not None]
errors: List[str] = []
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index ca043bf..f4ee01a 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -1,4 +1,5 @@
"""OpenCV based camera backend."""
+
from __future__ import annotations
import time
@@ -21,15 +22,13 @@ def open(self) -> None:
backend_flag = self._resolve_backend(self.settings.properties.get("api"))
self._capture = cv2.VideoCapture(int(self.settings.index), backend_flag)
if not self._capture.isOpened():
- raise RuntimeError(
- f"Unable to open camera index {self.settings.index} with OpenCV"
- )
+ raise RuntimeError(f"Unable to open camera index {self.settings.index} with OpenCV")
self._configure_capture()
def read(self) -> Tuple[np.ndarray, float]:
if self._capture is None:
raise RuntimeError("Camera has not been opened")
-
+
# Try grab first - this is non-blocking and helps detect connection issues faster
grabbed = self._capture.grab()
if not grabbed:
@@ -38,12 +37,12 @@ def read(self) -> Tuple[np.ndarray, float]:
raise RuntimeError("OpenCV camera connection lost")
# Otherwise treat as temporary frame read failure (timeout-like)
raise TimeoutError("Failed to grab frame from OpenCV camera (temporary)")
-
+
# Now retrieve the frame
success, frame = self._capture.retrieve()
if not success or frame is None:
raise TimeoutError("Failed to retrieve frame from OpenCV camera (temporary)")
-
+
return frame, time.time()
def close(self) -> None:
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index ca1f3e5..126eb13 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -1,10 +1,11 @@
"""Configuration helpers for the DLC Live GUI."""
+
from __future__ import annotations
+import json
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any, Dict, Optional
-import json
@dataclass
@@ -30,12 +31,12 @@ def apply_defaults(self) -> "CameraSettings":
self.fps = float(self.fps) if self.fps else 30.0
self.exposure = int(self.exposure) if self.exposure else 0
self.gain = float(self.gain) if self.gain else 0.0
- self.crop_x0 = max(0, int(self.crop_x0)) if hasattr(self, 'crop_x0') else 0
- self.crop_y0 = max(0, int(self.crop_y0)) if hasattr(self, 'crop_y0') else 0
- self.crop_x1 = max(0, int(self.crop_x1)) if hasattr(self, 'crop_x1') else 0
- self.crop_y1 = max(0, int(self.crop_y1)) if hasattr(self, 'crop_y1') else 0
+ self.crop_x0 = max(0, int(self.crop_x0)) if hasattr(self, "crop_x0") else 0
+ self.crop_y0 = max(0, int(self.crop_y0)) if hasattr(self, "crop_y0") else 0
+ self.crop_x1 = max(0, int(self.crop_x1)) if hasattr(self, "crop_x1") else 0
+ self.crop_y1 = max(0, int(self.crop_y1)) if hasattr(self, "crop_y1") else 0
return self
-
+
def get_crop_region(self) -> Optional[tuple[int, int, int, int]]:
"""Get crop region as (x0, y0, x1, y1) or None if no cropping."""
if self.crop_x0 == 0 and self.crop_y0 == 0 and self.crop_x1 == 0 and self.crop_y1 == 0:
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 6358c74..80944d8 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -1,4 +1,5 @@
"""DLCLive integration helpers."""
+
from __future__ import annotations
import logging
@@ -31,6 +32,7 @@ class PoseResult:
@dataclass
class ProcessorStats:
"""Statistics for DLC processor performance."""
+
frames_enqueued: int = 0
frames_processed: int = 0
frames_dropped: int = 0
@@ -59,7 +61,7 @@ def __init__(self) -> None:
self._worker_thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
self._initialized = False
-
+
# Statistics tracking
self._frames_enqueued = 0
self._frames_processed = 0
@@ -94,11 +96,11 @@ def enqueue_frame(self, frame: np.ndarray, timestamp: float) -> None:
# Start worker thread with initialization
self._start_worker(frame.copy(), timestamp)
return
-
+
# Don't count dropped frames until processor is initialized
if not self._initialized:
return
-
+
if self._queue is not None:
try:
# Non-blocking put - drop frame if queue is full
@@ -113,22 +115,20 @@ def enqueue_frame(self, frame: np.ndarray, timestamp: float) -> None:
def get_stats(self) -> ProcessorStats:
"""Get current processing statistics."""
queue_size = self._queue.qsize() if self._queue is not None else 0
-
+
with self._stats_lock:
- avg_latency = (
- sum(self._latencies) / len(self._latencies)
- if self._latencies
- else 0.0
- )
+ avg_latency = sum(self._latencies) / len(self._latencies) if self._latencies else 0.0
last_latency = self._latencies[-1] if self._latencies else 0.0
-
+
# Compute processing FPS from processing times
if len(self._processing_times) >= 2:
duration = self._processing_times[-1] - self._processing_times[0]
- processing_fps = (len(self._processing_times) - 1) / duration if duration > 0 else 0.0
+ processing_fps = (
+ (len(self._processing_times) - 1) / duration if duration > 0 else 0.0
+ )
else:
processing_fps = 0.0
-
+
return ProcessorStats(
frames_enqueued=self._frames_enqueued,
frames_processed=self._frames_processed,
@@ -142,7 +142,7 @@ def get_stats(self) -> ProcessorStats:
def _start_worker(self, init_frame: np.ndarray, init_timestamp: float) -> None:
if self._worker_thread is not None and self._worker_thread.is_alive():
return
-
+
self._queue = queue.Queue(maxsize=2)
self._stop_event.clear()
self._worker_thread = threading.Thread(
@@ -156,18 +156,18 @@ def _start_worker(self, init_frame: np.ndarray, init_timestamp: float) -> None:
def _stop_worker(self) -> None:
if self._worker_thread is None:
return
-
+
self._stop_event.set()
if self._queue is not None:
try:
self._queue.put_nowait(_SENTINEL)
except queue.Full:
pass
-
+
self._worker_thread.join(timeout=2.0)
if self._worker_thread.is_alive():
LOGGER.warning("DLC worker thread did not terminate cleanly")
-
+
self._worker_thread = None
self._queue = None
@@ -175,17 +175,15 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
try:
# Initialize model
if DLCLive is None:
- raise RuntimeError(
- "The 'dlclive' package is required for pose estimation."
- )
+ raise RuntimeError("The 'dlclive' package is required for pose estimation.")
if not self._settings.model_path:
raise RuntimeError("No DLCLive model path configured.")
-
+
options = {
"model_path": self._settings.model_path,
"model_type": self._settings.model_type,
"processor": self._processor,
- "dynamic": [False,0.5,10],
+ "dynamic": [False, 0.5, 10],
"resize": 1.0,
}
# todo expose more parameters from settings
@@ -194,53 +192,53 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
self._initialized = True
self.initialized.emit(True)
LOGGER.info("DLCLive model initialized successfully")
-
+
# Process the initialization frame
enqueue_time = time.perf_counter()
pose = self._dlc.get_pose(init_frame, frame_time=init_timestamp)
process_time = time.perf_counter()
-
+
with self._stats_lock:
self._frames_enqueued += 1
self._frames_processed += 1
self._processing_times.append(process_time)
-
+
self.pose_ready.emit(PoseResult(pose=pose, timestamp=init_timestamp))
-
+
except Exception as exc:
LOGGER.exception("Failed to initialize DLCLive", exc_info=exc)
self.error.emit(str(exc))
self.initialized.emit(False)
return
-
+
# Main processing loop
while not self._stop_event.is_set():
try:
item = self._queue.get(timeout=0.1)
except queue.Empty:
continue
-
+
if item is _SENTINEL:
break
-
+
frame, timestamp, enqueue_time = item
try:
start_process = time.perf_counter()
pose = self._dlc.get_pose(frame, frame_time=timestamp)
end_process = time.perf_counter()
-
+
latency = end_process - enqueue_time
-
+
with self._stats_lock:
self._frames_processed += 1
self._latencies.append(latency)
self._processing_times.append(end_process)
-
+
self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
except Exception as exc:
LOGGER.exception("Pose inference failed", exc_info=exc)
self.error.emit(str(exc))
finally:
self._queue.task_done()
-
+
LOGGER.info("DLC worker thread exiting")
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index 66db93c..bd3bee7 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -1,11 +1,12 @@
"""PyQt6 based GUI for DeepLabCut Live."""
+
from __future__ import annotations
-import os
import json
+import logging
+import os
import sys
import time
-import logging
from collections import deque
from pathlib import Path
from typing import Optional
@@ -18,6 +19,7 @@
QApplication,
QCheckBox,
QComboBox,
+ QDoubleSpinBox,
QFileDialog,
QFormLayout,
QGroupBox,
@@ -30,7 +32,6 @@
QPushButton,
QSizePolicy,
QSpinBox,
- QDoubleSpinBox,
QStatusBar,
QVBoxLayout,
QWidget,
@@ -40,22 +41,24 @@
from dlclivegui.cameras import CameraFactory
from dlclivegui.cameras.factory import DetectedCamera
from dlclivegui.config import (
+ DEFAULT_CONFIG,
ApplicationSettings,
BoundingBoxSettings,
CameraSettings,
DLCProcessorSettings,
RecordingSettings,
- DEFAULT_CONFIG,
)
from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult, ProcessorStats
-from dlclivegui.processors.processor_utils import scan_processor_folder, instantiate_from_scan
+from dlclivegui.processors.processor_utils import instantiate_from_scan, scan_processor_folder
from dlclivegui.video_recorder import RecorderStats, VideoRecorder
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ["CUDA_VISIBLE_DEVICES"] = "0"
logging.basicConfig(level=logging.INFO)
PATH2MODELS = "C:\\Users\\User\\Repos\\DeepLabCut-live-GUI\\models"
+
+
class MainWindow(QMainWindow):
"""Main application window."""
@@ -112,16 +115,12 @@ def _setup_ui(self) -> None:
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
- self.video_label.setSizePolicy(
- QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding
- )
+ self.video_label.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
# Controls panel with fixed width to prevent shifting
controls_widget = QWidget()
controls_widget.setMaximumWidth(500)
- controls_widget.setSizePolicy(
- QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding
- )
+ controls_widget.setSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Expanding)
controls_layout = QVBoxLayout(controls_widget)
controls_layout.setContentsMargins(5, 5, 5, 5)
controls_layout.addWidget(self._build_camera_group())
@@ -218,25 +217,25 @@ def _build_camera_group(self) -> QGroupBox:
self.crop_x0.setPrefix("x0:")
self.crop_x0.setSpecialValueText("x0:None")
crop_layout.addWidget(self.crop_x0)
-
+
self.crop_y0 = QSpinBox()
self.crop_y0.setRange(0, 4320)
self.crop_y0.setPrefix("y0:")
self.crop_y0.setSpecialValueText("y0:None")
crop_layout.addWidget(self.crop_y0)
-
+
self.crop_x1 = QSpinBox()
self.crop_x1.setRange(0, 7680)
self.crop_x1.setPrefix("x1:")
self.crop_x1.setSpecialValueText("x1:None")
crop_layout.addWidget(self.crop_x1)
-
+
self.crop_y1 = QSpinBox()
self.crop_y1.setRange(0, 4320)
self.crop_y1.setPrefix("y1:")
self.crop_y1.setSpecialValueText("y1:None")
crop_layout.addWidget(self.crop_y1)
-
+
form.addRow("Crop (x0,y0,x1,y1)", crop_layout)
self.rotation_combo = QComboBox()
@@ -275,22 +274,22 @@ def _build_dlc_group(self) -> QGroupBox:
self.processor_folder_edit = QLineEdit()
self.processor_folder_edit.setText(str(Path(__file__).parent.joinpath("processors")))
processor_path_layout.addWidget(self.processor_folder_edit)
-
+
self.browse_processor_folder_button = QPushButton("Browse...")
self.browse_processor_folder_button.clicked.connect(self._action_browse_processor_folder)
processor_path_layout.addWidget(self.browse_processor_folder_button)
-
+
self.refresh_processors_button = QPushButton("Refresh")
self.refresh_processors_button.clicked.connect(self._refresh_processors)
processor_path_layout.addWidget(self.refresh_processors_button)
form.addRow("Processor folder", processor_path_layout)
-
+
self.processor_combo = QComboBox()
self.processor_combo.addItem("No Processor", None)
form.addRow("Processor", self.processor_combo)
self.additional_options_edit = QPlainTextEdit()
- self.additional_options_edit.setPlaceholderText('')
+ self.additional_options_edit.setPlaceholderText("")
self.additional_options_edit.setFixedHeight(40)
form.addRow("Additional options", self.additional_options_edit)
@@ -314,7 +313,9 @@ def _build_dlc_group(self) -> QGroupBox:
self.auto_record_checkbox = QCheckBox("Auto-record video on processor command")
self.auto_record_checkbox.setChecked(False)
- self.auto_record_checkbox.setToolTip("Automatically start/stop video recording when processor receives video recording commands")
+ self.auto_record_checkbox.setToolTip(
+ "Automatically start/stop video recording when processor receives video recording commands"
+ )
form.addRow(self.auto_record_checkbox)
self.processor_status_label = QLabel("Processor: No clients | Recording: No")
@@ -386,31 +387,31 @@ def _build_bbox_group(self) -> QGroupBox:
form.addRow(self.bbox_enabled_checkbox)
bbox_layout = QHBoxLayout()
-
+
self.bbox_x0_spin = QSpinBox()
self.bbox_x0_spin.setRange(0, 7680)
self.bbox_x0_spin.setPrefix("x0:")
self.bbox_x0_spin.setValue(0)
bbox_layout.addWidget(self.bbox_x0_spin)
-
+
self.bbox_y0_spin = QSpinBox()
self.bbox_y0_spin.setRange(0, 4320)
self.bbox_y0_spin.setPrefix("y0:")
self.bbox_y0_spin.setValue(0)
bbox_layout.addWidget(self.bbox_y0_spin)
-
+
self.bbox_x1_spin = QSpinBox()
self.bbox_x1_spin.setRange(0, 7680)
self.bbox_x1_spin.setPrefix("x1:")
self.bbox_x1_spin.setValue(100)
bbox_layout.addWidget(self.bbox_x1_spin)
-
+
self.bbox_y1_spin = QSpinBox()
self.bbox_y1_spin.setRange(0, 4320)
self.bbox_y1_spin.setPrefix("y1:")
self.bbox_y1_spin.setValue(100)
bbox_layout.addWidget(self.bbox_y1_spin)
-
+
form.addRow("Coordinates", bbox_layout)
return group
@@ -429,10 +430,8 @@ def _connect_signals(self) -> None:
self.rotation_combo.currentIndexChanged.connect(self._on_rotation_changed)
self.start_inference_button.clicked.connect(self._start_inference)
self.stop_inference_button.clicked.connect(lambda: self._stop_inference())
- self.show_predictions_checkbox.stateChanged.connect(
- self._on_show_predictions_changed
- )
-
+ self.show_predictions_checkbox.stateChanged.connect(self._on_show_predictions_changed)
+
# Connect bounding box controls
self.bbox_enabled_checkbox.stateChanged.connect(self._on_bbox_changed)
self.bbox_x0_spin.valueChanged.connect(self._on_bbox_changed)
@@ -454,17 +453,17 @@ def _connect_signals(self) -> None:
def _apply_config(self, config: ApplicationSettings) -> None:
camera = config.camera
self.camera_fps.setValue(float(camera.fps))
-
+
# Set exposure and gain from config
self.camera_exposure.setValue(int(camera.exposure))
self.camera_gain.setValue(float(camera.gain))
-
+
# Set crop settings from config
- self.crop_x0.setValue(int(camera.crop_x0) if hasattr(camera, 'crop_x0') else 0)
- self.crop_y0.setValue(int(camera.crop_y0) if hasattr(camera, 'crop_y0') else 0)
- self.crop_x1.setValue(int(camera.crop_x1) if hasattr(camera, 'crop_x1') else 0)
- self.crop_y1.setValue(int(camera.crop_y1) if hasattr(camera, 'crop_y1') else 0)
-
+ self.crop_x0.setValue(int(camera.crop_x0) if hasattr(camera, "crop_x0") else 0)
+ self.crop_y0.setValue(int(camera.crop_y0) if hasattr(camera, "crop_y0") else 0)
+ self.crop_x1.setValue(int(camera.crop_x1) if hasattr(camera, "crop_x1") else 0)
+ self.crop_y1.setValue(int(camera.crop_y1) if hasattr(camera, "crop_y1") else 0)
+
backend_name = camera.backend or "opencv"
self.camera_backend.blockSignals(True)
index = self.camera_backend.findData(backend_name)
@@ -474,25 +473,21 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.camera_backend.setEditText(backend_name)
self.camera_backend.blockSignals(False)
self._refresh_camera_indices(keep_current=False)
- self._select_camera_by_index(
- camera.index, fallback_text=camera.name or str(camera.index)
- )
-
+ self._select_camera_by_index(camera.index, fallback_text=camera.name or str(camera.index))
+
self._active_camera_settings = None
self._update_backend_specific_controls()
dlc = config.dlc
self.model_path_edit.setText(dlc.model_path)
-
+
# Set model type
model_type = dlc.model_type or "base"
model_type_index = self.model_type_combo.findData(model_type)
if model_type_index >= 0:
self.model_type_combo.setCurrentIndex(model_type_index)
-
- self.additional_options_edit.setPlainText(
- json.dumps(dlc.additional_options, indent=2)
- )
+
+ self.additional_options_edit.setPlainText(json.dumps(dlc.additional_options, indent=2))
recording = config.recording
self.output_directory_edit.setText(recording.directory)
@@ -527,17 +522,17 @@ def _camera_settings_from_ui(self) -> CameraSettings:
if index is None:
raise ValueError("Camera selection must provide a numeric index")
backend_text = self._current_backend_name()
-
+
# Get exposure and gain from explicit UI fields
exposure = self.camera_exposure.value()
gain = self.camera_gain.value()
-
+
# Get crop settings from UI
crop_x0 = self.crop_x0.value()
crop_y0 = self.crop_y0.value()
crop_x1 = self.crop_x1.value()
crop_y1 = self.crop_y1.value()
-
+
name_text = self.camera_index.currentText().strip()
settings = CameraSettings(
name=name_text or f"Camera {index}",
@@ -561,17 +556,15 @@ def _current_backend_name(self) -> str:
text = self.camera_backend.currentText().strip()
return text or "opencv"
- def _refresh_camera_indices(
- self, *_args: object, keep_current: bool = True
- ) -> None:
+ def _refresh_camera_indices(self, *_args: object, keep_current: bool = True) -> None:
backend = self._current_backend_name()
# Get max_devices from config, default to 3
- max_devices = self._config.camera.max_devices if hasattr(self._config.camera, 'max_devices') else 3
+ max_devices = (
+ self._config.camera.max_devices if hasattr(self._config.camera, "max_devices") else 3
+ )
detected = CameraFactory.detect_cameras(backend, max_devices=max_devices)
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
- logging.info(
- f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
- )
+ logging.info(f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}")
self._detected_cameras = detected
previous_index = self._current_camera_index_value()
previous_text = self.camera_index.currentText()
@@ -590,9 +583,7 @@ def _refresh_camera_indices(
self.camera_index.setEditText("")
self.camera_index.blockSignals(False)
- def _select_camera_by_index(
- self, index: int, fallback_text: Optional[str] = None
- ) -> None:
+ def _select_camera_by_index(self, index: int, fallback_text: Optional[str] = None) -> None:
self.camera_index.blockSignals(True)
for row in range(self.camera_index.count()):
if self.camera_index.itemData(row) == index:
@@ -615,9 +606,7 @@ def _current_camera_index_value(self) -> Optional[int]:
except ValueError:
return None
debug_info = [f"{camera.index}:{camera.label}" for camera in detected]
- logging.info(
- f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}"
- )
+ logging.info(f"[CameraDetection] Available cameras for backend '{backend}': {debug_info}")
self._detected_cameras = detected
previous_index = self._current_camera_index_value()
previous_text = self.camera_index.currentText()
@@ -636,9 +625,7 @@ def _current_camera_index_value(self) -> Optional[int]:
self.camera_index.setEditText("")
self.camera_index.blockSignals(False)
- def _select_camera_by_index(
- self, index: int, fallback_text: Optional[str] = None
- ) -> None:
+ def _select_camera_by_index(self, index: int, fallback_text: Optional[str] = None) -> None:
self.camera_index.blockSignals(True)
for row in range(self.camera_index.count()):
if self.camera_index.itemData(row) == index:
@@ -671,13 +658,11 @@ def _dlc_settings_from_ui(self) -> DLCProcessorSettings:
model_type = self.model_type_combo.currentData()
if not isinstance(model_type, str):
model_type = "base"
-
+
return DLCProcessorSettings(
model_path=self.model_path_edit.text().strip(),
model_type=model_type,
- additional_options=self._parse_json(
- self.additional_options_edit.toPlainText()
- ),
+ additional_options=self._parse_json(self.additional_options_edit.toPlainText()),
)
def _recording_settings_from_ui(self) -> RecordingSettings:
@@ -760,9 +745,7 @@ def _action_browse_directory(self) -> None:
def _action_browse_processor_folder(self) -> None:
"""Browse for processor folder."""
current_path = self.processor_folder_edit.text() or "./processors"
- directory = QFileDialog.getExistingDirectory(
- self, "Select processor folder", current_path
- )
+ directory = QFileDialog.getExistingDirectory(self, "Select processor folder", current_path)
if directory:
self.processor_folder_edit.setText(directory)
self._refresh_processors()
@@ -770,25 +753,25 @@ def _action_browse_processor_folder(self) -> None:
def _refresh_processors(self) -> None:
"""Scan processor folder and populate dropdown."""
folder_path = self.processor_folder_edit.text() or "./processors"
-
+
# Clear existing items (keep "No Processor")
self.processor_combo.clear()
self.processor_combo.addItem("No Processor", None)
-
+
# Scan folder
try:
self._scanned_processors = scan_processor_folder(folder_path)
self._processor_keys = list(self._scanned_processors.keys())
-
+
# Populate dropdown
for key in self._processor_keys:
info = self._scanned_processors[key]
display_name = f"{info['name']} ({info['file']})"
self.processor_combo.addItem(display_name, key)
-
+
status_msg = f"Found {len(self._processor_keys)} processor(s) in {folder_path}"
self.statusBar().showMessage(status_msg, 3000)
-
+
except Exception as e:
error_msg = f"Error scanning processors: {e}"
self.statusBar().showMessage(error_msg, 5000)
@@ -803,11 +786,11 @@ def _update_backend_specific_controls(self) -> None:
"""Enable/disable controls based on selected backend."""
backend = self._current_backend_name()
is_opencv = backend.lower() == "opencv"
-
+
# Disable exposure and gain controls for OpenCV backend
self.camera_exposure.setEnabled(not is_opencv)
self.camera_gain.setEnabled(not is_opencv)
-
+
# Set tooltip to explain why controls are disabled
if is_opencv:
tooltip = "Exposure and gain control not supported with OpenCV backend"
@@ -877,9 +860,7 @@ def _on_camera_started(self, settings: CameraSettings) -> None:
fps_text = f"{float(settings.fps):.2f} FPS"
else:
fps_text = "unknown FPS"
- self.statusBar().showMessage(
- f"Camera preview started @ {fps_text}", 5000
- )
+ self.statusBar().showMessage(f"Camera preview started @ {fps_text}", 5000)
self._update_inference_buttons()
self._update_camera_controls_enabled()
@@ -912,7 +893,7 @@ def _configure_dlc(self) -> bool:
if not settings.model_path:
self._show_error("Please select a DLCLive model before starting inference.")
return False
-
+
# Instantiate processor if selected
processor = None
selected_key = self.processor_combo.currentData()
@@ -920,16 +901,16 @@ def _configure_dlc(self) -> bool:
try:
# For now, instantiate with no parameters
# TODO: Add parameter dialog for processors that need params
- # or pass kwargs from config ?
+ # or pass kwargs from config ?
processor = instantiate_from_scan(self._scanned_processors, selected_key)
- processor_name = self._scanned_processors[selected_key]['name']
+ processor_name = self._scanned_processors[selected_key]["name"]
self.statusBar().showMessage(f"Loaded processor: {processor_name}", 3000)
except Exception as e:
error_msg = f"Failed to instantiate processor: {e}"
self._show_error(error_msg)
logging.error(error_msg)
return False
-
+
self.dlc_processor.configure(settings, processor=processor)
return True
@@ -955,9 +936,7 @@ def _update_dlc_controls_enabled(self) -> None:
widget.setEnabled(allow_changes)
def _update_camera_controls_enabled(self) -> None:
- recording_active = (
- self._video_recorder is not None and self._video_recorder.is_running
- )
+ recording_active = self._video_recorder is not None and self._video_recorder.is_running
allow_changes = (
not self.camera_controller.is_running()
and not self._dlc_active
@@ -1043,7 +1022,7 @@ def _update_metrics(self) -> None:
self.camera_stats_label.setText("Measuring…")
else:
self.camera_stats_label.setText("Camera idle")
-
+
if hasattr(self, "dlc_stats_label"):
if self._dlc_active and self._dlc_initialized:
stats = self.dlc_processor.get_stats()
@@ -1051,11 +1030,11 @@ def _update_metrics(self) -> None:
self.dlc_stats_label.setText(summary)
else:
self.dlc_stats_label.setText("DLC processor idle")
-
+
# Update processor status (connection and recording state)
if hasattr(self, "processor_status_label"):
self._update_processor_status()
-
+
if hasattr(self, "recording_stats_label"):
if self._video_recorder is not None:
stats = self._video_recorder.get_stats()
@@ -1074,47 +1053,49 @@ def _update_processor_status(self) -> None:
if not self._dlc_active or not self._dlc_initialized:
self.processor_status_label.setText("Processor: Not active")
return
-
+
# Get processor instance from dlc_processor
processor = self.dlc_processor._processor
-
+
if processor is None:
self.processor_status_label.setText("Processor: None loaded")
return
-
+
# Check if processor has the required attributes (socket-based processors)
- if not hasattr(processor, 'conns') or not hasattr(processor, '_recording'):
+ if not hasattr(processor, "conns") or not hasattr(processor, "_recording"):
self.processor_status_label.setText("Processor: No status info")
return
-
+
# Get connection count and recording state
num_clients = len(processor.conns)
- is_recording = processor.recording if hasattr(processor, 'recording') else False
-
+ is_recording = processor.recording if hasattr(processor, "recording") else False
+
# Format status message
client_str = f"{num_clients} client{'s' if num_clients != 1 else ''}"
recording_str = "Yes" if is_recording else "No"
self.processor_status_label.setText(f"Clients: {client_str} | Recording: {recording_str}")
-
+
# Handle auto-recording based on processor's video recording flag
- if hasattr(processor, '_vid_recording') and self.auto_record_checkbox.isChecked():
+ if hasattr(processor, "_vid_recording") and self.auto_record_checkbox.isChecked():
current_vid_recording = processor.video_recording
-
+
# Check if video recording state changed
if current_vid_recording != self._last_processor_vid_recording:
if current_vid_recording:
# Start video recording
if not self._video_recorder or not self._video_recorder.is_running:
# Get session name from processor
- session_name = getattr(processor, 'session_name', 'auto_session')
+ session_name = getattr(processor, "session_name", "auto_session")
self._auto_record_session_name = session_name
-
+
# Update filename with session name
original_filename = self.filename_edit.text()
self.filename_edit.setText(f"{session_name}.mp4")
-
+
self._start_recording()
- self.statusBar().showMessage(f"Auto-started recording: {session_name}", 3000)
+ self.statusBar().showMessage(
+ f"Auto-started recording: {session_name}", 3000
+ )
logging.info(f"Auto-recording started for session: {session_name}")
else:
# Stop video recording
@@ -1122,7 +1103,7 @@ def _update_processor_status(self) -> None:
self._stop_recording()
self.statusBar().showMessage("Auto-stopped recording", 3000)
logging.info("Auto-recording stopped")
-
+
self._last_processor_vid_recording = current_vid_recording
def _start_inference(self) -> None:
@@ -1130,9 +1111,7 @@ def _start_inference(self) -> None:
self.statusBar().showMessage("Pose inference already running", 3000)
return
if not self.camera_controller.is_running():
- self._show_error(
- "Start the camera preview before running pose inference."
- )
+ self._show_error("Start the camera preview before running pose inference.")
return
if not self._configure_dlc():
self._update_inference_buttons()
@@ -1141,13 +1120,13 @@ def _start_inference(self) -> None:
self._last_pose = None
self._dlc_active = True
self._dlc_initialized = False
-
+
# Update button to show initializing state
self.start_inference_button.setText("Initializing DLCLive!")
self.start_inference_button.setStyleSheet("background-color: #4A90E2; color: white;")
self.start_inference_button.setEnabled(False)
self.stop_inference_button.setEnabled(True)
-
+
self.statusBar().showMessage("Initializing DLCLive…", 3000)
self._update_camera_controls_enabled()
self._update_dlc_controls_enabled()
@@ -1160,11 +1139,11 @@ def _stop_inference(self, show_message: bool = True) -> None:
self._last_pose = None
self._last_processor_vid_recording = False
self._auto_record_session_name = None
-
+
# Reset button appearance
self.start_inference_button.setText("Start pose inference")
self.start_inference_button.setStyleSheet("")
-
+
if self._current_frame is not None:
self._display_frame(self._current_frame, force=True)
if was_active and show_message:
@@ -1236,9 +1215,7 @@ def _stop_recording(self) -> None:
self.recording_stats_label.setText(summary)
else:
self._last_recorder_summary = (
- self._format_recorder_stats(stats)
- if stats is not None
- else "Recorder idle"
+ self._format_recorder_stats(stats) if stats is not None else "Recorder idle"
)
self._last_drop_warning = 0.0
self.statusBar().showMessage("Recording stopped", 3000)
@@ -1248,10 +1225,10 @@ def _stop_recording(self) -> None:
def _on_frame_ready(self, frame_data: FrameData) -> None:
raw_frame = frame_data.image
self._raw_frame = raw_frame
-
+
# Apply cropping before rotation
frame = self._apply_crop(raw_frame)
-
+
# Apply rotation
frame = self._apply_rotation(frame)
frame = np.ascontiguousarray(frame)
@@ -1263,9 +1240,7 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
if not success:
now = time.perf_counter()
if now - self._last_drop_warning > 1.0:
- self.statusBar().showMessage(
- "Recorder backlog full; dropping frames", 2000
- )
+ self.statusBar().showMessage("Recorder backlog full; dropping frames", 2000)
self._last_drop_warning = now
except RuntimeError as exc:
# Check if it's a frame size error
@@ -1290,7 +1265,7 @@ def _on_pose_ready(self, result: PoseResult) -> None:
if not self._dlc_active:
return
self._last_pose = result
- #logging.info(f"Pose result: {result.pose}, Timestamp: {result.timestamp}")
+ # logging.info(f"Pose result: {result.pose}, Timestamp: {result.timestamp}")
if self._current_frame is not None:
self._display_frame(self._current_frame, force=True)
@@ -1306,11 +1281,11 @@ def _update_video_display(self, frame: np.ndarray) -> None:
and self._last_pose.pose is not None
):
display_frame = self._draw_pose(frame, self._last_pose.pose)
-
+
# Draw bounding box if enabled
if self._bbox_enabled:
display_frame = self._draw_bbox(display_frame)
-
+
rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB)
h, w, ch = rgb.shape
bytes_per_line = ch * w
@@ -1321,20 +1296,20 @@ def _apply_crop(self, frame: np.ndarray) -> np.ndarray:
"""Apply cropping to the frame based on settings."""
if self._active_camera_settings is None:
return frame
-
+
crop_region = self._active_camera_settings.get_crop_region()
if crop_region is None:
return frame
-
+
x0, y0, x1, y1 = crop_region
height, width = frame.shape[:2]
-
+
# Validate and constrain crop coordinates
x0 = max(0, min(x0, width))
y0 = max(0, min(y0, height))
x1 = max(x0, min(x1, width)) if x1 > 0 else width
y1 = max(y0, min(y1, height)) if y1 > 0 else height
-
+
# Apply crop
if x0 < x1 and y0 < y1:
return frame[y0:y1, x0:x1]
@@ -1362,7 +1337,7 @@ def _on_bbox_changed(self, _value: int = 0) -> None:
self._bbox_y0 = self.bbox_y0_spin.value()
self._bbox_x1 = self.bbox_x1_spin.value()
self._bbox_y1 = self.bbox_y1_spin.value()
-
+
# Force redraw if preview is running
if self._current_frame is not None:
self._display_frame(self._current_frame, force=True)
@@ -1374,20 +1349,20 @@ def _draw_bbox(self, frame: np.ndarray) -> np.ndarray:
y0 = self._bbox_y0
x1 = self._bbox_x1
y1 = self._bbox_y1
-
+
# Validate coordinates
if x0 >= x1 or y0 >= y1:
return overlay
-
+
height, width = frame.shape[:2]
x0 = max(0, min(x0, width - 1))
y0 = max(0, min(y0, height - 1))
x1 = max(x0 + 1, min(x1, width))
y1 = max(y0 + 1, min(y1, height))
-
+
# Draw red rectangle (BGR format: red is (0, 0, 255))
cv2.rectangle(overlay, (x0, y0), (x1, y1), (0, 0, 255), 2)
-
+
return overlay
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
diff --git a/dlclivegui/processors/GUI_INTEGRATION.md b/dlclivegui/processors/GUI_INTEGRATION.md
index 446232e..6e504f1 100644
--- a/dlclivegui/processors/GUI_INTEGRATION.md
+++ b/dlclivegui/processors/GUI_INTEGRATION.md
@@ -97,13 +97,13 @@ dropdown.set_items(display_names)
def on_processor_selected(dropdown_index):
# Get the key
key = self.processor_keys[dropdown_index]
-
+
# Get processor info
info = all_processors[key]
-
+
# Show description
description_label.text = info['description']
-
+
# Build parameter form
for param_name, param_info in info['params'].items():
add_parameter_field(
@@ -119,17 +119,17 @@ def on_processor_selected(dropdown_index):
def on_create_clicked():
# Get selected key
key = self.processor_keys[dropdown.current_index]
-
+
# Get user's parameter values
user_params = parameter_form.get_values()
-
+
# Instantiate using the key!
self.processor = instantiate_from_scan(
all_processors,
key,
**user_params
)
-
+
print(f"Created: {self.processor.__class__.__name__}")
```
diff --git a/dlclivegui/processors/PLUGIN_SYSTEM.md b/dlclivegui/processors/PLUGIN_SYSTEM.md
index b02402d..fc9cab4 100644
--- a/dlclivegui/processors/PLUGIN_SYSTEM.md
+++ b/dlclivegui/processors/PLUGIN_SYSTEM.md
@@ -47,7 +47,7 @@ Two helper functions enable GUI discovery:
```python
def get_available_processors():
"""Returns dict of available processors with metadata."""
-
+
def instantiate_processor(class_name, **kwargs):
"""Instantiates a processor by name with given parameters."""
```
@@ -96,7 +96,7 @@ def load_processors_from_file(file_path):
spec = importlib.util.spec_from_file_location("processors", file_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
-
+
if hasattr(module, 'get_available_processors'):
return module.get_available_processors()
return {}
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index 3f5b951..fb6522c 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -65,7 +65,7 @@ class BaseProcessor_socket(Processor):
Handles network connections, timing, and data logging.
Subclasses should implement custom pose processing logic.
"""
-
+
# Metadata for GUI discovery
PROCESSOR_NAME = "Base Socket Processor"
PROCESSOR_DESCRIPTION = "Base class for socket-based processors with multi-client support"
@@ -73,23 +73,23 @@ class BaseProcessor_socket(Processor):
"bind": {
"type": "tuple",
"default": ("0.0.0.0", 6000),
- "description": "Server address (host, port)"
+ "description": "Server address (host, port)",
},
"authkey": {
"type": "bytes",
"default": b"secret password",
- "description": "Authentication key for clients"
+ "description": "Authentication key for clients",
},
"use_perf_counter": {
"type": "bool",
"default": False,
- "description": "Use time.perf_counter() instead of time.time()"
+ "description": "Use time.perf_counter() instead of time.time()",
},
"save_original": {
"type": "bool",
"default": False,
- "description": "Save raw pose arrays for analysis"
- }
+ "description": "Save raw pose arrays for analysis",
+ },
}
def __init__(
@@ -139,12 +139,12 @@ def __init__(
# State
self.curr_step = 0
self.save_original = save_original
-
+
@property
def recording(self):
"""Thread-safe recording flag."""
return self._recording.is_set()
-
+
@property
def video_recording(self):
"""Thread-safe video recording flag."""
@@ -153,7 +153,7 @@ def video_recording(self):
@property
def session_name(self):
return self._session_name
-
+
@session_name.setter
def session_name(self, name):
self._session_name = name
@@ -188,18 +188,18 @@ def _rx_loop(self, c):
pass
self.conns.discard(c)
LOG.info("Client disconnected")
-
+
def _handle_client_message(self, msg):
"""Handle control messages from clients."""
if not isinstance(msg, dict):
return
-
+
cmd = msg.get("cmd")
if cmd == "set_session_name":
session_name = msg.get("session_name", "default_session")
self.session_name = session_name
LOG.info(f"Session name set to: {session_name}")
-
+
elif cmd == "start_recording":
self._vid_recording.set()
self._recording.set()
@@ -207,12 +207,12 @@ def _handle_client_message(self, msg):
self._clear_data_queues()
self.curr_step = 0
LOG.info("Recording started, data queues cleared")
-
+
elif cmd == "stop_recording":
self._recording.clear()
self._vid_recording.clear()
LOG.info("Recording stopped")
-
+
elif cmd == "save":
filename = msg.get("filename", self.filename)
save_code = self.save(filename)
@@ -222,7 +222,7 @@ def _handle_client_message(self, msg):
# Placeholder for video recording start
self._vid_recording.set()
LOG.info("Start video recording command received")
-
+
def _clear_data_queues(self):
"""Clear all data storage queues. Override in subclasses to clear additional queues."""
self.time_stamp.clear()
@@ -338,41 +338,43 @@ class MyProcessor_socket(BaseProcessor_socket):
Broadcasts: [timestamp, center_x, center_y, heading, head_angle]
"""
-
+
# Metadata for GUI discovery
PROCESSOR_NAME = "Mouse Pose Processor"
- PROCESSOR_DESCRIPTION = "Calculates mouse center, heading, and head angle with optional One-Euro filtering"
+ PROCESSOR_DESCRIPTION = (
+ "Calculates mouse center, heading, and head angle with optional One-Euro filtering"
+ )
PROCESSOR_PARAMS = {
"bind": {
"type": "tuple",
"default": ("0.0.0.0", 6000),
- "description": "Server address (host, port)"
+ "description": "Server address (host, port)",
},
"authkey": {
"type": "bytes",
"default": b"secret password",
- "description": "Authentication key for clients"
+ "description": "Authentication key for clients",
},
"use_perf_counter": {
"type": "bool",
"default": False,
- "description": "Use time.perf_counter() instead of time.time()"
+ "description": "Use time.perf_counter() instead of time.time()",
},
"use_filter": {
"type": "bool",
"default": False,
- "description": "Apply One-Euro filter to calculated values"
+ "description": "Apply One-Euro filter to calculated values",
},
"filter_kwargs": {
"type": "dict",
"default": {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0},
- "description": "One-Euro filter parameters (min_cutoff, beta, d_cutoff)"
+ "description": "One-Euro filter parameters (min_cutoff, beta, d_cutoff)",
},
"save_original": {
"type": "bool",
"default": False,
- "description": "Save raw pose arrays for analysis"
- }
+ "description": "Save raw pose arrays for analysis",
+ },
}
def __init__(
@@ -542,7 +544,7 @@ def get_data(self):
def get_available_processors():
"""
Get list of available processor classes.
-
+
Returns:
dict: Dictionary mapping class names to processor info:
{
@@ -560,7 +562,7 @@ def get_available_processors():
"class": processor_class,
"name": getattr(processor_class, "PROCESSOR_NAME", class_name),
"description": getattr(processor_class, "PROCESSOR_DESCRIPTION", ""),
- "params": getattr(processor_class, "PROCESSOR_PARAMS", {})
+ "params": getattr(processor_class, "PROCESSOR_PARAMS", {}),
}
return processors
@@ -568,20 +570,20 @@ def get_available_processors():
def instantiate_processor(class_name, **kwargs):
"""
Instantiate a processor by class name with given parameters.
-
+
Args:
class_name: Name of the processor class (e.g., "MyProcessor_socket")
**kwargs: Parameters to pass to the processor constructor
-
+
Returns:
Processor instance
-
+
Raises:
ValueError: If class_name is not in registry
"""
if class_name not in PROCESSOR_REGISTRY:
available = ", ".join(PROCESSOR_REGISTRY.keys())
raise ValueError(f"Unknown processor '{class_name}'. Available: {available}")
-
+
processor_class = PROCESSOR_REGISTRY[class_name]
return processor_class(**kwargs)
diff --git a/dlclivegui/processors/processor_utils.py b/dlclivegui/processors/processor_utils.py
index dcacaa4..b69b3a7 100644
--- a/dlclivegui/processors/processor_utils.py
+++ b/dlclivegui/processors/processor_utils.py
@@ -1,4 +1,3 @@
-
import importlib.util
import inspect
from pathlib import Path
@@ -7,10 +6,10 @@
def load_processors_from_file(file_path):
"""
Load all processor classes from a Python file.
-
+
Args:
file_path: Path to Python file containing processors
-
+
Returns:/home/as153/work_geneva/mice_ar_tasks/mouse_ar/ctrl/dlc_processors/GUI_INTEGRATION.md
dict: Dictionary of available processors
"""
@@ -18,13 +17,14 @@ def load_processors_from_file(file_path):
spec = importlib.util.spec_from_file_location("processors", file_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
-
+
# Check if module has get_available_processors function
- if hasattr(module, 'get_available_processors'):
+ if hasattr(module, "get_available_processors"):
return module.get_available_processors()
-
+
# Fallback: scan for Processor subclasses
from dlclive import Processor
+
processors = {}
for name, obj in inspect.getmembers(module, inspect.isclass):
if issubclass(obj, Processor) and obj != Processor:
@@ -32,7 +32,7 @@ def load_processors_from_file(file_path):
"class": obj,
"name": getattr(obj, "PROCESSOR_NAME", name),
"description": getattr(obj, "PROCESSOR_DESCRIPTION", ""),
- "params": getattr(obj, "PROCESSOR_PARAMS", {})
+ "params": getattr(obj, "PROCESSOR_PARAMS", {}),
}
return processors
@@ -40,10 +40,10 @@ def load_processors_from_file(file_path):
def scan_processor_folder(folder_path):
"""
Scan a folder for all Python files with processor definitions.
-
+
Args:
folder_path: Path to folder containing processor files
-
+
Returns:
dict: Dictionary mapping unique processor keys to processor info:
{
@@ -59,11 +59,11 @@ def scan_processor_folder(folder_path):
"""
all_processors = {}
folder = Path(folder_path)
-
+
for py_file in folder.glob("*.py"):
if py_file.name.startswith("_"):
continue
-
+
try:
processors = load_processors_from_file(py_file)
for class_name, processor_info in processors.items():
@@ -76,26 +76,26 @@ def scan_processor_folder(folder_path):
all_processors[key] = processor_info
except Exception as e:
print(f"Error loading {py_file}: {e}")
-
+
return all_processors
def instantiate_from_scan(processors_dict, processor_key, **kwargs):
"""
Instantiate a processor from scan_processor_folder results.
-
+
Args:
processors_dict: Dict returned by scan_processor_folder
processor_key: Key like "file.py::ClassName"
**kwargs: Parameters for processor constructor
-
+
Returns:
Processor instance
-
+
Example:
processors = scan_processor_folder("./dlc_processors")
processor = instantiate_from_scan(
- processors,
+ processors,
"dlc_processor_socket.py::MyProcessor_socket",
use_filter=True
)
@@ -103,7 +103,7 @@ def instantiate_from_scan(processors_dict, processor_key, **kwargs):
if processor_key not in processors_dict:
available = ", ".join(processors_dict.keys())
raise ValueError(f"Unknown processor '{processor_key}'. Available: {available}")
-
+
processor_info = processors_dict[processor_key]
processor_class = processor_info["class"]
return processor_class(**kwargs)
@@ -111,17 +111,16 @@ def instantiate_from_scan(processors_dict, processor_key, **kwargs):
def display_processor_info(processors):
"""Display processor information in a user-friendly format."""
- print("\n" + "="*70)
+ print("\n" + "=" * 70)
print("AVAILABLE PROCESSORS")
- print("="*70)
-
+ print("=" * 70)
+
for idx, (class_name, info) in enumerate(processors.items(), 1):
print(f"\n[{idx}] {info['name']}")
print(f" Class: {class_name}")
print(f" Description: {info['description']}")
print(f" Parameters:")
- for param_name, param_info in info['params'].items():
+ for param_name, param_info in info["params"].items():
print(f" - {param_name} ({param_info['type']})")
print(f" Default: {param_info['default']}")
print(f" {param_info['description']}")
-
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index 2190314..a40ed28 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -1,4 +1,5 @@
"""Video recording support using the vidgear library."""
+
from __future__ import annotations
import json
@@ -113,9 +114,7 @@ def start(self) -> None:
)
self._writer_thread.start()
- def configure_stream(
- self, frame_size: Tuple[int, int], frame_rate: Optional[float]
- ) -> None:
+ def configure_stream(self, frame_size: Tuple[int, int], frame_rate: Optional[float]) -> None:
self._frame_size = frame_size
self._frame_rate = frame_rate
@@ -125,13 +124,13 @@ def write(self, frame: np.ndarray, timestamp: Optional[float] = None) -> bool:
error = self._current_error()
if error is not None:
raise RuntimeError(f"Video encoding failed: {error}") from error
-
+
# Record timestamp for this frame
if timestamp is None:
timestamp = time.time()
with self._stats_lock:
self._frame_timestamps.append(timestamp)
-
+
# Convert frame to uint8 if needed
if frame.dtype != np.uint8:
frame_float = frame.astype(np.float32, copy=False)
@@ -140,14 +139,14 @@ def write(self, frame: np.ndarray, timestamp: Optional[float] = None) -> bool:
if max_val > 0:
scale = 255.0 / max_val if max_val > 255.0 else (255.0 if max_val <= 1.0 else 1.0)
frame = np.clip(frame_float * scale, 0.0, 255.0).astype(np.uint8)
-
+
# Convert grayscale to RGB if needed
if frame.ndim == 2:
frame = np.repeat(frame[:, :, None], 3, axis=2)
-
+
# Ensure contiguous array
frame = np.ascontiguousarray(frame)
-
+
# Check if frame size matches expected size
if self._frame_size is not None:
expected_h, expected_w = self._frame_size
@@ -164,7 +163,7 @@ def write(self, frame: np.ndarray, timestamp: Optional[float] = None) -> bool:
f"Frame size changed from (h={expected_h}, w={expected_w}) to (h={actual_h}, w={actual_w})"
)
return False
-
+
try:
assert self._queue is not None
self._queue.put(frame, block=False)
@@ -200,10 +199,10 @@ def stop(self) -> None:
self._writer.close()
except Exception:
logger.exception("Failed to close WriteGear cleanly")
-
+
# Save timestamps to JSON file
self._save_timestamps()
-
+
self._writer = None
self._writer_thread = None
self._queue = None
@@ -224,9 +223,7 @@ def get_stats(self) -> Optional[RecorderStats]:
frames_written = self._frames_written
dropped = self._dropped_frames
avg_latency = (
- self._total_latency / self._frames_written
- if self._frames_written
- else 0.0
+ self._total_latency / self._frames_written if self._frames_written else 0.0
)
last_latency = self._last_latency
write_fps = self._compute_write_fps_locked()
@@ -312,14 +309,16 @@ def _save_timestamps(self) -> None:
if not self._frame_timestamps:
logger.info("No timestamps to save")
return
-
+
# Create timestamps file path
- timestamp_file = self._output.with_suffix('').with_suffix(self._output.suffix + '_timestamps.json')
-
+ timestamp_file = self._output.with_suffix("").with_suffix(
+ self._output.suffix + "_timestamps.json"
+ )
+
try:
with self._stats_lock:
timestamps = self._frame_timestamps.copy()
-
+
# Prepare metadata
data = {
"video_file": str(self._output.name),
@@ -329,11 +328,11 @@ def _save_timestamps(self) -> None:
"end_time": timestamps[-1] if timestamps else None,
"duration_seconds": timestamps[-1] - timestamps[0] if len(timestamps) > 1 else 0.0,
}
-
+
# Write to JSON
- with open(timestamp_file, 'w') as f:
+ with open(timestamp_file, "w") as f:
json.dump(data, f, indent=2)
-
+
logger.info(f"Saved {len(timestamps)} frame timestamps to {timestamp_file}")
except Exception as exc:
logger.exception(f"Failed to save timestamps to {timestamp_file}: {exc}")
diff --git a/setup.py b/setup.py
index a254101..02954f2 100644
--- a/setup.py
+++ b/setup.py
@@ -1,4 +1,5 @@
"""Setup configuration for the DeepLabCut Live GUI."""
+
from __future__ import annotations
import setuptools
From 39be1b20b2ed1102237612398816e3970ac7a2fc Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 24 Oct 2025 16:20:00 +0200
Subject: [PATCH 14/26] documentations
---
.pre-commit-config.yaml | 2 +-
README.md | 371 ++++++++++++---
dlclivegui/cameras/aravis_backend.py | 323 +++++++++++++
dlclivegui/cameras/factory.py | 3 +-
docs/README.md | 262 +++++++++++
docs/aravis_backend.md | 202 +++++++++
docs/camera_support.md | 84 +++-
docs/features.md | 653 +++++++++++++++++++++++++++
docs/install.md | 2 +-
docs/timestamp_format.md | 79 ++++
docs/user_guide.md | 633 ++++++++++++++++++++++++++
11 files changed, 2544 insertions(+), 70 deletions(-)
create mode 100644 dlclivegui/cameras/aravis_backend.py
create mode 100644 docs/README.md
create mode 100644 docs/aravis_backend.md
create mode 100644 docs/features.md
create mode 100644 docs/timestamp_format.md
create mode 100644 docs/user_guide.md
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 09dc3cd..0178c8e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ repos:
- id: check-yaml
- id: end-of-file-fixer
- id: name-tests-test
- args: [--pytest-test-first]
+ args: [--pytest-test-first]
- id: trailing-whitespace
- id: check-merge-conflict
diff --git a/README.md b/README.md
index a886a98..34a2682 100644
--- a/README.md
+++ b/README.md
@@ -1,126 +1,375 @@
# DeepLabCut Live GUI
-A modernised PyQt6 GUI for running [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) experiments. The application
-streams frames from a camera, optionally performs DLCLive inference, and records video using the
-[vidgear](https://github.com/abhiTronix/vidgear) toolkit.
+A modern PyQt6 GUI for running [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) experiments with real-time pose estimation. The application streams frames from industrial or consumer cameras, performs DLCLive inference, and records high-quality video with synchronized pose data.
## Features
-- Python 3.11+ compatible codebase with a PyQt6 interface.
-- Modular architecture with dedicated modules for camera control, video recording, configuration
- management, and DLCLive processing.
-- Single JSON configuration file that captures camera settings, DLCLive parameters, and recording
- options. All fields can be edited directly within the GUI.
-- Optional DLCLive inference with pose visualisation over the live video feed.
-- Recording support via vidgear's `WriteGear`, including custom encoder options.
+### Core Functionality
+- **Modern Python Stack**: Python 3.10+ compatible codebase with PyQt6 interface
+- **Multi-Backend Camera Support**: OpenCV, GenTL (Harvesters), Aravis, and Basler (pypylon)
+- **Real-Time Pose Estimation**: Live DLCLive inference with configurable models (TensorFlow, PyTorch)
+- **High-Performance Recording**: Hardware-accelerated video encoding via FFmpeg
+- **Flexible Configuration**: Single JSON file for all settings with GUI editing
+
+### Camera Features
+- **Multiple Backends**:
+ - OpenCV - Universal webcam support
+ - GenTL - Industrial cameras via Harvesters (Windows/Linux)
+ - Aravis - GenICam/GigE cameras (Linux/macOS)
+ - Basler - Basler cameras via pypylon
+- **Smart Device Detection**: Automatic camera enumeration without unnecessary probing
+- **Camera Controls**: Exposure time, gain, frame rate, and ROI cropping
+- **Live Preview**: Real-time camera feed with rotation support (0°, 90°, 180°, 270°)
+
+### DLCLive Features
+- **Model Support**: TensorFlow (base) and PyTorch models
+- **Processor System**: Plugin architecture for custom pose processing
+- **Auto-Recording**: Automatic video recording triggered by processor commands
+- **Performance Metrics**: Real-time FPS, latency, and queue monitoring
+- **Pose Visualization**: Optional overlay of detected keypoints on live feed
+
+### Recording Features
+- **Hardware Encoding**: NVENC (NVIDIA GPU) and software codecs (libx264, libx265)
+- **Configurable Quality**: CRF-based quality control
+- **Multiple Formats**: MP4, AVI, MOV containers
+- **Timestamp Support**: Frame-accurate timestamps for synchronization
+- **Performance Monitoring**: Write FPS, buffer status, and dropped frame tracking
+
+### User Interface
+- **Intuitive Layout**: Organized control panels with clear separation of concerns
+- **Configuration Management**: Load/save settings, support for multiple configurations
+- **Status Indicators**: Real-time feedback on camera, inference, and recording status
+- **Bounding Box Tool**: Visual overlay for ROI definition
## Installation
-1. Install the package and its dependencies:
+### Basic Installation
- ```bash
- pip install deeplabcut-live-gui
- ```
+```bash
+pip install deeplabcut-live-gui
+```
+
+This installs the core package with OpenCV camera support.
+
+### Full Installation with Optional Dependencies
+
+```bash
+# Install with gentl support
+pip install deeplabcut-live-gui[gentl]
+```
+
+### Platform-Specific Camera Backend Setup
+
+#### Windows (GenTL for Industrial Cameras)
+1. Install camera vendor drivers and SDK
+2. Ensure GenTL producer (.cti) files are accessible
+3. Common locations:
+ - `C:\Program Files\The Imaging Source Europe GmbH\IC4 GenTL Driver\bin\`
+ - Check vendor documentation for CTI file location
+
+#### Linux (Aravis for GenICam Cameras - Recommended)
+NOT tested
+```bash
+# Ubuntu/Debian
+sudo apt-get install gir1.2-aravis-0.8 python3-gi
+
+# Fedora
+sudo dnf install aravis python3-gobject
+```
+
+#### macOS (Aravis)
+NOT tested
+```bash
+brew install aravis
+pip install pygobject
+```
- The GUI requires additional runtime packages for optional features:
+#### Basler Cameras (All Platforms)
+NOT tested
+```bash
+# Install Pylon SDK from Basler website
+# Then install pypylon
+pip install pypylon
+```
- - [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live) for pose estimation.
- - [vidgear](https://github.com/abhiTronix/vidgear) for video recording.
- - [OpenCV](https://opencv.org/) for camera access.
+### Hardware Acceleration (Optional)
- These libraries are listed in `setup.py` and will be installed automatically when the package is
- installed via `pip`.
+For NVIDIA GPU encoding (highly recommended for high-resolution/high-FPS recording):
+```bash
+# Ensure NVIDIA drivers are installed
+# FFmpeg with NVENC support will be used automatically
+```
-2. Launch the GUI:
+## Quick Start
+1. **Launch the GUI**:
```bash
dlclivegui
```
+2. **Select Camera Backend**: Choose from the dropdown (opencv, gentl, aravis, basler)
+
+3. **Configure Camera**: Set FPS, exposure, gain, and other parameters
+
+4. **Start Preview**: Click "Start Preview" to begin camera streaming
+
+5. **Optional - Load DLC Model**: Browse to your exported DLCLive model directory
+
+6. **Optional - Start Inference**: Click "Start pose inference" for real-time tracking
+
+7. **Optional - Record Video**: Configure output path and click "Start recording"
+
## Configuration
-The GUI works with a single JSON configuration describing the experiment. The configuration contains
-three main sections:
+The GUI uses a single JSON configuration file containing all experiment settings:
```json
{
"camera": {
+ "name": "Camera 0",
"index": 0,
- "width": 1280,
- "height": 720,
"fps": 60.0,
- "backend": "opencv",
+ "backend": "gentl",
+ "exposure": 10000,
+ "gain": 5.0,
+ "crop_x0": 0,
+ "crop_y0": 0,
+ "crop_x1": 0,
+ "crop_y1": 0,
+ "max_devices": 3,
"properties": {}
},
"dlc": {
"model_path": "/path/to/exported-model",
- "processor": "cpu",
- "shuffle": 1,
- "trainingsetindex": 0,
- "processor_args": {},
- "additional_options": {}
+ "model_type": "base",
+ "additional_options": {
+ "resize": 0.5,
+ "processor": "cpu"
+ }
},
"recording": {
"enabled": true,
- "directory": "~/Videos/deeplabcut",
+ "directory": "~/Videos/deeplabcut-live",
"filename": "session.mp4",
"container": "mp4",
- "options": {
- "compression_mode": "mp4"
- }
+ "codec": "h264_nvenc",
+ "crf": 23
+ },
+ "bbox": {
+ "enabled": false,
+ "x0": 0,
+ "y0": 0,
+ "x1": 200,
+ "y1": 100
}
}
```
-Use **File → Load configuration…** to open an existing configuration, or **File → Save configuration**
-to persist the current settings. Every field in the GUI is editable, and values entered in the
-interface will be written back to the JSON file.
+### Configuration Management
-### Camera backends
+- **Load**: File → Load configuration… (or Ctrl+O)
+- **Save**: File → Save configuration (or Ctrl+S)
+- **Save As**: File → Save configuration as… (or Ctrl+Shift+S)
-Set `camera.backend` to one of the supported drivers:
+All GUI fields are automatically synchronized with the configuration file.
-- `opencv` – standard `cv2.VideoCapture` fallback available on every platform.
-- `basler` – uses the Basler Pylon SDK via `pypylon` (install separately).
-- `gentl` – uses Aravis for GenTL-compatible cameras (requires `python-gi` bindings).
+## Camera Backends
-Backend specific parameters can be supplied through the `camera.properties` object. For example:
+### Backend Selection Guide
+| Backend | Platform | Use Case | Auto-Detection |
+|---------|----------|----------|----------------|
+| **opencv** | All | Webcams, simple USB cameras | Basic |
+| **gentl** | Windows, Linux | Industrial cameras via CTI files | Yes |
+| **aravis** | Linux, macOS | GenICam/GigE cameras | Yes |
+| **basler** | All | Basler cameras specifically | Yes |
+
+### Backend-Specific Configuration
+
+#### OpenCV
+```json
+{
+ "camera": {
+ "backend": "opencv",
+ "index": 0,
+ "fps": 30.0
+ }
+}
+```
+**Note**: Exposure and gain controls are disabled for OpenCV backend due to limited driver support.
+
+#### GenTL (Harvesters)
```json
{
"camera": {
+ "backend": "gentl",
"index": 0,
- "backend": "basler",
+ "fps": 60.0,
+ "exposure": 15000,
+ "gain": 8.0,
"properties": {
- "serial": "40123456",
- "exposure": 15000,
- "gain": 6.0
+ "cti_file": "C:\\Path\\To\\Producer.cti",
+ "serial_number": "12345678",
+ "pixel_format": "Mono8"
}
}
}
```
-If optional dependencies are missing, the GUI will show the backend as unavailable in the drop-down
-but you can still configure it for a system where the drivers are present.
+#### Aravis
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0,
+ "fps": 60.0,
+ "exposure": 10000,
+ "gain": 5.0,
+ "properties": {
+ "camera_id": "TheImagingSource-12345678",
+ "pixel_format": "Mono8",
+ "n_buffers": 10,
+ "timeout": 2000000
+ }
+ }
+}
+```
+
+See [Camera Backend Documentation](docs/camera_support.md) for detailed setup instructions.
+
+## DLCLive Integration
-## Development
+### Model Types
-The core modules of the package are organised as follows:
+The GUI supports both TensorFlow and PyTorch DLCLive models:
-- `dlclivegui.config` – dataclasses for loading, storing, and saving application settings.
-- `dlclivegui.cameras` – modular camera backends (OpenCV, Basler, GenTL) and factory helpers.
-- `dlclivegui.camera_controller` – camera capture worker running in a dedicated `QThread`.
-- `dlclivegui.video_recorder` – wrapper around `WriteGear` for video output.
-- `dlclivegui.dlc_processor` – asynchronous DLCLive inference with optional pose overlay.
-- `dlclivegui.gui` – PyQt6 user interface and application entry point.
+1. **Base (TensorFlow)**: Original DLC models exported for live inference
+2. **PyTorch**: PyTorch-based models (requires PyTorch installation)
-Run a quick syntax check with:
+Select the model type from the dropdown before starting inference.
+
+### Processor System
+
+The GUI includes a plugin system for custom pose processing:
+
+```python
+# Example processor
+class MyProcessor:
+ def process(self, pose, timestamp):
+ # Custom processing logic
+ x, y = pose[0, :2] # First keypoint
+ print(f"Position: ({x}, {y})")
+ def save(self):
+ pass
+```
+
+Place processors in `dlclivegui/processors/` and refresh to load them.
+
+See [Processor Plugin Documentation](docs/PLUGIN_SYSTEM.md) for details.
+
+### Auto-Recording Feature
+
+Enable "Auto-record video on processor command" to automatically start/stop recording based on processor signals. Useful for event-triggered recording in behavioral experiments.
+
+## Performance Optimization
+
+### High-Speed Camera Tips
+
+1. **Use Hardware Encoding**: Select `h264_nvenc` codec for NVIDIA GPUs
+2. **Adjust Buffer Count**: Increase buffers for GenTL/Aravis backends
+ ```json
+ "properties": {"n_buffers": 20}
+ ```
+3. **Optimize CRF**: Lower CRF = higher quality but larger files (default: 23)
+4. **Disable Visualization**: Uncheck "Display pose predictions" during recording
+5. **Crop Region**: Use cropping to reduce frame size before inference
+
+### Recommended Settings by FPS
+
+| FPS Range | Codec | CRF | Buffers | Notes |
+|-----------|-------|-----|---------|-------|
+| 30-60 | libx264 | 23 | 10 | Standard quality |
+| 60-120 | h264_nvenc | 23 | 15 | GPU encoding |
+| 120-200 | h264_nvenc | 28 | 20 | Higher compression |
+| 200+ | h264_nvenc | 30 | 30 | Max performance |
+
+### Project Structure
+
+```
+dlclivegui/
+├── __init__.py
+├── gui.py # Main PyQt6 application
+├── config.py # Configuration dataclasses
+├── camera_controller.py # Camera capture thread
+├── dlc_processor.py # DLCLive inference thread
+├── video_recorder.py # Video encoding thread
+├── cameras/ # Camera backend modules
+│ ├── base.py # Abstract base class
+│ ├── factory.py # Backend registry and detection
+│ ├── opencv_backend.py
+│ ├── gentl_backend.py
+│ ├── aravis_backend.py
+│ └── basler_backend.py
+└── processors/ # Pose processor plugins
+ ├── processor_utils.py
+ └── dlc_processor_socket.py
+```
+
+### Running Tests
```bash
+# Syntax check
python -m compileall dlclivegui
+
+# Type checking (optional)
+mypy dlclivegui
+
```
+### Adding New Camera Backends
+
+1. Create new backend inheriting from `CameraBackend`
+2. Implement required methods: `open()`, `read()`, `close()`
+3. Optional: Implement `get_device_count()` for smart detection
+4. Register in `cameras/factory.py`
+
+See [Camera Backend Development](docs/camera_support.md) for detailed instructions.
+
+
+## Documentation
+
+- [Camera Support](docs/camera_support.md) - All camera backends and setup
+- [Aravis Backend](docs/aravis_backend.md) - GenICam camera setup (Linux/macOS)
+- [Processor Plugins](docs/PLUGIN_SYSTEM.md) - Custom pose processing
+- [Installation Guide](docs/install.md) - Detailed setup instructions
+- [Timestamp Format](docs/timestamp_format.md) - Timestamp synchronization
+
+## System Requirements
+
+
+### Recommended
+- Python 3.10+
+- 8 GB RAM
+- NVIDIA GPU with CUDA support (for DLCLive inference and video encoding)
+- USB 3.0 or GigE network (for industrial cameras)
+- SSD storage (for high-speed recording)
+
+### Tested Platforms
+- Windows 11
+
## License
-This project is licensed under the GNU Lesser General Public License v3.0. See the `LICENSE` file for
-more information.
+This project is licensed under the GNU Lesser General Public License v3.0. See the [LICENSE](LICENSE) file for more information.
+
+## Citation
+
+Cite the original DeepLabCut-live paper:
+```bibtex
+@article{Kane2020,
+ title={Real-time, low-latency closed-loop feedback using markerless posture tracking},
+ author={Kane, Gary A and Lopes, Gonçalo and Saunders, Jonny L and Mathis, Alexander and Mathis, Mackenzie W},
+ journal={eLife},
+ year={2020},
+ doi={10.7554/eLife.61909}
+}
+```
diff --git a/dlclivegui/cameras/aravis_backend.py b/dlclivegui/cameras/aravis_backend.py
new file mode 100644
index 0000000..e033096
--- /dev/null
+++ b/dlclivegui/cameras/aravis_backend.py
@@ -0,0 +1,323 @@
+"""Aravis backend for GenICam cameras."""
+
+from __future__ import annotations
+
+import time
+from typing import Optional, Tuple
+
+import cv2
+import numpy as np
+
+from .base import CameraBackend
+
+try: # pragma: no cover - optional dependency
+ import gi
+
+ gi.require_version("Aravis", "0.8")
+ from gi.repository import Aravis
+
+ ARAVIS_AVAILABLE = True
+except Exception: # pragma: no cover - optional dependency
+ Aravis = None # type: ignore
+ ARAVIS_AVAILABLE = False
+
+
+class AravisCameraBackend(CameraBackend):
+ """Capture frames from GenICam-compatible devices via Aravis."""
+
+ def __init__(self, settings):
+ super().__init__(settings)
+ props = settings.properties
+ self._camera_id: Optional[str] = props.get("camera_id")
+ self._pixel_format: str = props.get("pixel_format", "Mono8")
+ self._timeout: int = int(props.get("timeout", 2000000)) # microseconds
+ self._n_buffers: int = int(props.get("n_buffers", 10))
+
+ self._camera = None
+ self._stream = None
+ self._device_label: Optional[str] = None
+
+ @classmethod
+ def is_available(cls) -> bool:
+ """Check if Aravis is available on this system."""
+ return ARAVIS_AVAILABLE
+
+ @classmethod
+ def get_device_count(cls) -> int:
+ """Get the actual number of Aravis devices detected.
+
+ Returns the number of devices found, or -1 if detection fails.
+ """
+ if not ARAVIS_AVAILABLE:
+ return -1
+
+ try:
+ Aravis.update_device_list()
+ return Aravis.get_n_devices()
+ except Exception:
+ return -1
+
+ def open(self) -> None:
+ """Open the Aravis camera device."""
+ if not ARAVIS_AVAILABLE: # pragma: no cover - optional dependency
+ raise RuntimeError(
+ "The 'aravis' library is required for the Aravis backend. "
+ "Install it via your system package manager (e.g., 'sudo apt install gir1.2-aravis-0.8' on Ubuntu)."
+ )
+
+ # Update device list
+ Aravis.update_device_list()
+ n_devices = Aravis.get_n_devices()
+
+ if n_devices == 0:
+ raise RuntimeError("No Aravis cameras detected")
+
+ # Open camera by ID or index
+ if self._camera_id:
+ self._camera = Aravis.Camera.new(self._camera_id)
+ if self._camera is None:
+ raise RuntimeError(f"Failed to open camera with ID '{self._camera_id}'")
+ else:
+ index = int(self.settings.index or 0)
+ if index < 0 or index >= n_devices:
+ raise RuntimeError(
+ f"Camera index {index} out of range for {n_devices} Aravis device(s)"
+ )
+ camera_id = Aravis.get_device_id(index)
+ self._camera = Aravis.Camera.new(camera_id)
+ if self._camera is None:
+ raise RuntimeError(f"Failed to open camera at index {index}")
+
+ # Get device information for label
+ self._device_label = self._resolve_device_label()
+
+ # Configure camera
+ self._configure_pixel_format()
+ self._configure_exposure()
+ self._configure_gain()
+ self._configure_frame_rate()
+
+ # Create stream
+ self._stream = self._camera.create_stream(None, None)
+ if self._stream is None:
+ raise RuntimeError("Failed to create Aravis stream")
+
+ # Push buffers to stream
+ payload_size = self._camera.get_payload()
+ for _ in range(self._n_buffers):
+ self._stream.push_buffer(Aravis.Buffer.new_allocate(payload_size))
+
+ # Start acquisition
+ self._camera.start_acquisition()
+
+ def read(self) -> Tuple[np.ndarray, float]:
+ """Read a frame from the camera."""
+ if self._camera is None or self._stream is None:
+ raise RuntimeError("Aravis camera not initialized")
+
+ # Pop buffer from stream
+ buffer = self._stream.timeout_pop_buffer(self._timeout)
+
+ if buffer is None:
+ raise TimeoutError("Failed to grab frame from Aravis camera (timeout)")
+
+ # Check buffer status
+ status = buffer.get_status()
+ if status != Aravis.BufferStatus.SUCCESS:
+ self._stream.push_buffer(buffer)
+ raise TimeoutError(f"Aravis buffer status error: {status}")
+
+ # Get image data
+ try:
+ # Get buffer data as numpy array
+ data = buffer.get_data()
+ width = buffer.get_image_width()
+ height = buffer.get_image_height()
+ pixel_format = buffer.get_image_pixel_format()
+
+ # Convert to numpy array
+ if pixel_format == Aravis.PIXEL_FORMAT_MONO_8:
+ frame = np.frombuffer(data, dtype=np.uint8).reshape((height, width))
+ frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
+ elif pixel_format == Aravis.PIXEL_FORMAT_RGB_8_PACKED:
+ frame = np.frombuffer(data, dtype=np.uint8).reshape((height, width, 3))
+ frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
+ elif pixel_format == Aravis.PIXEL_FORMAT_BGR_8_PACKED:
+ frame = np.frombuffer(data, dtype=np.uint8).reshape((height, width, 3))
+ elif pixel_format in (Aravis.PIXEL_FORMAT_MONO_12, Aravis.PIXEL_FORMAT_MONO_16):
+ # Handle 12-bit and 16-bit mono
+ frame = np.frombuffer(data, dtype=np.uint16).reshape((height, width))
+ # Scale to 8-bit
+ max_val = float(frame.max()) if frame.size else 0.0
+ scale = 255.0 / max_val if max_val > 0.0 else 1.0
+ frame = np.clip(frame * scale, 0, 255).astype(np.uint8)
+ frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
+ else:
+ # Fallback for unknown formats - try to interpret as mono8
+ frame = np.frombuffer(data, dtype=np.uint8).reshape((height, width))
+ frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
+
+ frame = frame.copy()
+ timestamp = time.time()
+
+ finally:
+ # Always push buffer back to stream
+ self._stream.push_buffer(buffer)
+
+ return frame, timestamp
+
+ def stop(self) -> None:
+ """Stop camera acquisition."""
+ if self._camera is not None:
+ try:
+ self._camera.stop_acquisition()
+ except Exception:
+ pass
+
+ def close(self) -> None:
+ """Release the camera and stream."""
+ if self._camera is not None:
+ try:
+ self._camera.stop_acquisition()
+ except Exception:
+ pass
+
+ # Clear stream buffers
+ if self._stream is not None:
+ try:
+ # Flush remaining buffers
+ while True:
+ buffer = self._stream.try_pop_buffer()
+ if buffer is None:
+ break
+ except Exception:
+ pass
+ self._stream = None
+
+ # Release camera
+ try:
+ del self._camera
+ except Exception:
+ pass
+ finally:
+ self._camera = None
+
+ self._device_label = None
+
+ def device_name(self) -> str:
+ """Return a human-readable device name."""
+ if self._device_label:
+ return self._device_label
+ return super().device_name()
+
+ # ------------------------------------------------------------------
+ # Configuration helpers
+ # ------------------------------------------------------------------
+
+ def _configure_pixel_format(self) -> None:
+ """Configure the camera pixel format."""
+ if self._camera is None:
+ return
+
+ try:
+ # Map common format names to Aravis pixel formats
+ format_map = {
+ "Mono8": Aravis.PIXEL_FORMAT_MONO_8,
+ "Mono12": Aravis.PIXEL_FORMAT_MONO_12,
+ "Mono16": Aravis.PIXEL_FORMAT_MONO_16,
+ "RGB8": Aravis.PIXEL_FORMAT_RGB_8_PACKED,
+ "BGR8": Aravis.PIXEL_FORMAT_BGR_8_PACKED,
+ }
+
+ if self._pixel_format in format_map:
+ self._camera.set_pixel_format(format_map[self._pixel_format])
+ else:
+ # Try setting as string
+ self._camera.set_pixel_format_from_string(self._pixel_format)
+ except Exception:
+ # If pixel format setting fails, continue with default
+ pass
+
+ def _configure_exposure(self) -> None:
+ """Configure camera exposure time."""
+ if self._camera is None:
+ return
+
+ # Get exposure from settings
+ exposure = None
+ if hasattr(self.settings, "exposure") and self.settings.exposure > 0:
+ exposure = float(self.settings.exposure)
+
+ if exposure is None:
+ return
+
+ try:
+ # Disable auto exposure
+ try:
+ self._camera.set_exposure_time_auto(Aravis.Auto.OFF)
+ except Exception:
+ pass
+
+ # Set exposure time (in microseconds)
+ self._camera.set_exposure_time(exposure)
+ except Exception:
+ pass
+
+ def _configure_gain(self) -> None:
+ """Configure camera gain."""
+ if self._camera is None:
+ return
+
+ # Get gain from settings
+ gain = None
+ if hasattr(self.settings, "gain") and self.settings.gain > 0.0:
+ gain = float(self.settings.gain)
+
+ if gain is None:
+ return
+
+ try:
+ # Disable auto gain
+ try:
+ self._camera.set_gain_auto(Aravis.Auto.OFF)
+ except Exception:
+ pass
+
+ # Set gain value
+ self._camera.set_gain(gain)
+ except Exception:
+ pass
+
+ def _configure_frame_rate(self) -> None:
+ """Configure camera frame rate."""
+ if self._camera is None or not self.settings.fps:
+ return
+
+ try:
+ target_fps = float(self.settings.fps)
+ self._camera.set_frame_rate(target_fps)
+ except Exception:
+ pass
+
+ def _resolve_device_label(self) -> Optional[str]:
+ """Get a human-readable device label."""
+ if self._camera is None:
+ return None
+
+ try:
+ model = self._camera.get_model_name()
+ vendor = self._camera.get_vendor_name()
+ serial = self._camera.get_device_serial_number()
+
+ if model and serial:
+ if vendor:
+ return f"{vendor} {model} ({serial})"
+ return f"{model} ({serial})"
+ elif model:
+ return model
+ elif serial:
+ return f"Camera {serial}"
+ except Exception:
+ pass
+
+ return None
diff --git a/dlclivegui/cameras/factory.py b/dlclivegui/cameras/factory.py
index eca4f58..3261ba5 100644
--- a/dlclivegui/cameras/factory.py
+++ b/dlclivegui/cameras/factory.py
@@ -22,6 +22,7 @@ class DetectedCamera:
"opencv": ("dlclivegui.cameras.opencv_backend", "OpenCVCameraBackend"),
"basler": ("dlclivegui.cameras.basler_backend", "BaslerCameraBackend"),
"gentl": ("dlclivegui.cameras.gentl_backend", "GenTLCameraBackend"),
+ "aravis": ("dlclivegui.cameras.aravis_backend", "AravisCameraBackend"),
}
@@ -58,7 +59,7 @@ def detect_cameras(backend: str, max_devices: int = 10) -> List[DetectedCamera]:
The backend identifier, e.g. ``"opencv"``.
max_devices:
Upper bound for the indices that should be probed.
- For GenTL backend, the actual device count is queried if available.
+ For backends with get_device_count (GenTL, Aravis), the actual device count is queried.
Returns
-------
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 0000000..3cc524f
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,262 @@
+# DeepLabCut-live-GUI Documentation Index
+
+Welcome to the DeepLabCut-live-GUI documentation! This index will help you find the information you need.
+
+## Getting Started
+
+### New Users
+1. **[README](../README.md)** - Project overview, installation, and quick start
+2. **[User Guide](user_guide.md)** - Step-by-step walkthrough of all features
+3. **[Installation Guide](install.md)** - Detailed installation instructions
+
+### Quick References
+- **[ARAVIS_QUICK_REF](../ARAVIS_QUICK_REF.md)** - Aravis backend quick reference
+- **[Features Overview](features.md)** - Complete feature documentation
+
+## Core Documentation
+
+### Camera Setup
+- **[Camera Support](camera_support.md)** - Overview of all camera backends
+- **[Aravis Backend](aravis_backend.md)** - Linux/macOS GenICam camera setup
+- Platform-specific guides for industrial cameras
+
+### Application Features
+- **[Features Documentation](features.md)** - Detailed feature descriptions:
+ - Camera control and backends
+ - Real-time pose estimation
+ - Video recording
+ - Configuration management
+ - Processor system
+ - User interface
+ - Performance monitoring
+ - Advanced features
+
+### User Guide
+- **[User Guide](user_guide.md)** - Complete usage walkthrough:
+ - Getting started
+ - Camera setup
+ - DLCLive configuration
+ - Recording videos
+ - Configuration management
+ - Common workflows
+ - Tips and best practices
+ - Troubleshooting
+
+## Advanced Topics
+
+### Processor System
+- **[Processor Plugins](PLUGIN_SYSTEM.md)** - Custom pose processing
+- **[Processor Auto-Recording](processor_auto_recording.md)** - Event-triggered recording
+- Socket processor documentation
+
+### Technical Details
+- **[Timestamp Format](timestamp_format.md)** - Synchronization and timing
+- **[ARAVIS_BACKEND_SUMMARY](../ARAVIS_BACKEND_SUMMARY.md)** - Implementation details
+
+## By Use Case
+
+### I want to...
+
+#### Set up a camera
+→ [Camera Support](camera_support.md) → Select backend → Follow setup guide
+
+**By Platform**:
+- **Windows**: [README](../README.md#windows-gentl-for-industrial-cameras) → GenTL setup
+- **Linux**: [Aravis Backend](aravis_backend.md) → Installation for Ubuntu/Debian
+- **macOS**: [Aravis Backend](aravis_backend.md) → Installation via Homebrew
+
+**By Camera Type**:
+- **Webcam**: [User Guide](user_guide.md#camera-setup) → OpenCV backend
+- **Industrial Camera**: [Camera Support](camera_support.md) → GenTL/Aravis
+- **Basler Camera**: [Camera Support](camera_support.md#basler-cameras) → pypylon setup
+- **The Imaging Source**: [Aravis Backend](aravis_backend.md) or GenTL
+
+#### Run pose estimation
+→ [User Guide](user_guide.md#dlclive-configuration) → Load model → Start inference
+
+#### Record high-speed video
+→ [Features](features.md#video-recording) → Hardware encoding → GPU setup
+→ [User Guide](user_guide.md#high-speed-recording-60-fps) → Optimization tips
+
+#### Create custom processor
+→ [Processor Plugins](PLUGIN_SYSTEM.md) → Plugin architecture → Examples
+
+#### Trigger recording remotely
+→ [Features](features.md#auto-recording-feature) → Auto-recording setup
+→ Socket processor documentation
+
+#### Optimize performance
+→ [Features](features.md#performance-optimization) → Metrics → Adjustments
+→ [User Guide](user_guide.md#tips-and-best-practices) → Best practices
+
+## By Topic
+
+### Camera Backends
+| Backend | Documentation | Platform |
+|---------|---------------|----------|
+| OpenCV | [User Guide](user_guide.md#step-1-select-camera-backend) | All |
+| GenTL | [Camera Support](camera_support.md) | Windows, Linux |
+| Aravis | [Aravis Backend](aravis_backend.md) | Linux, macOS |
+| Basler | [Camera Support](camera_support.md#basler-cameras) | All |
+
+### Configuration
+- **Basics**: [README](../README.md#configuration)
+- **Management**: [User Guide](user_guide.md#working-with-configurations)
+- **Templates**: [User Guide](user_guide.md#configuration-templates)
+- **Details**: [Features](features.md#configuration-management)
+
+### Recording
+- **Quick Start**: [User Guide](user_guide.md#recording-videos)
+- **Features**: [Features](features.md#video-recording)
+- **Optimization**: [README](../README.md#performance-optimization)
+- **Auto-Recording**: [Features](features.md#auto-recording-feature)
+
+### DLCLive
+- **Setup**: [User Guide](user_guide.md#dlclive-configuration)
+- **Models**: [Features](features.md#model-support)
+- **Performance**: [Features](features.md#performance-metrics)
+- **Visualization**: [Features](features.md#pose-visualization)
+
+## Troubleshooting
+
+### Quick Fixes
+1. **Camera not detected** → [User Guide](user_guide.md#troubleshooting-guide)
+2. **Slow inference** → [Features](features.md#performance-optimization)
+3. **Dropped frames** → [README](../README.md#troubleshooting)
+4. **Recording issues** → [User Guide](user_guide.md#troubleshooting-guide)
+
+### Detailed Troubleshooting
+- [User Guide - Troubleshooting Section](user_guide.md#troubleshooting-guide)
+- [README - Troubleshooting](../README.md#troubleshooting)
+- [Aravis Backend - Troubleshooting](aravis_backend.md#troubleshooting)
+
+## Development
+
+### Architecture
+- **Project Structure**: [README](../README.md#development)
+- **Backend Development**: [Camera Support](camera_support.md#contributing-new-camera-types)
+- **Processor Development**: [Processor Plugins](PLUGIN_SYSTEM.md)
+
+### Implementation Details
+- **Aravis Backend**: [ARAVIS_BACKEND_SUMMARY](../ARAVIS_BACKEND_SUMMARY.md)
+- **Thread Safety**: [Features](features.md#thread-safety)
+- **Resource Management**: [Features](features.md#resource-management)
+
+## Reference
+
+### Configuration Schema
+```json
+{
+ "camera": {
+ "name": "string",
+ "index": "number",
+ "fps": "number",
+ "backend": "opencv|gentl|aravis|basler",
+ "exposure": "number (μs, 0=auto)",
+ "gain": "number (0.0=auto)",
+ "crop_x0/y0/x1/y1": "number",
+ "max_devices": "number",
+ "properties": "object"
+ },
+ "dlc": {
+ "model_path": "string",
+ "model_type": "base|pytorch",
+ "additional_options": "object"
+ },
+ "recording": {
+ "enabled": "boolean",
+ "directory": "string",
+ "filename": "string",
+ "container": "mp4|avi|mov",
+ "codec": "h264_nvenc|libx264|hevc_nvenc",
+ "crf": "number (0-51)"
+ },
+ "bbox": {
+ "enabled": "boolean",
+ "x0/y0/x1/y1": "number"
+ }
+}
+```
+
+### Performance Metrics
+- **Camera FPS**: [Features](features.md#camera-metrics)
+- **DLC Metrics**: [Features](features.md#dlc-metrics)
+- **Recording Metrics**: [Features](features.md#recording-metrics)
+
+### Keyboard Shortcuts
+| Action | Shortcut |
+|--------|----------|
+| Load configuration | Ctrl+O |
+| Save configuration | Ctrl+S |
+| Save as | Ctrl+Shift+S |
+| Quit | Ctrl+Q |
+
+## External Resources
+
+### DeepLabCut
+- [DeepLabCut](http://www.mackenziemathislab.org/deeplabcut)
+- [DeepLabCut-live](https://github.com/DeepLabCut/DeepLabCut-live)
+- [DeepLabCut Documentation](http://deeplabcut.github.io/DeepLabCut/docs/intro.html)
+
+### Camera Libraries
+- [Aravis Project](https://github.com/AravisProject/aravis)
+- [Harvesters (GenTL)](https://github.com/genicam/harvesters)
+- [pypylon (Basler)](https://github.com/basler/pypylon)
+- [OpenCV](https://opencv.org/)
+
+### Video Encoding
+- [FFmpeg](https://ffmpeg.org/)
+- [NVENC (NVIDIA)](https://developer.nvidia.com/nvidia-video-codec-sdk)
+
+## Getting Help
+
+### Support Channels
+1. Check relevant documentation (use this index!)
+2. Search GitHub issues
+3. Review example configurations
+4. Contact maintainers
+
+### Reporting Issues
+When reporting bugs, include:
+- GUI version
+- Platform (OS, Python version)
+- Camera backend and model
+- Configuration file (if applicable)
+- Error messages
+- Steps to reproduce
+
+## Contributing
+
+Interested in contributing?
+- See [README - Contributing](../README.md#contributing)
+- Review [Development Section](../README.md#development)
+- Check open GitHub issues
+- Read coding guidelines
+
+---
+
+## Document Version History
+
+- **v1.0** - Initial comprehensive documentation
+ - Complete README overhaul
+ - User guide creation
+ - Features documentation
+ - Camera backend guides
+ - Aravis backend implementation
+
+## Quick Navigation
+
+**Popular Pages**:
+- [User Guide](user_guide.md) - Most comprehensive walkthrough
+- [Features](features.md) - All capabilities detailed
+- [Aravis Setup](aravis_backend.md) - Linux industrial cameras
+- [Camera Support](camera_support.md) - All camera backends
+
+**By Experience Level**:
+- **Beginner**: [README](../README.md) → [User Guide](user_guide.md)
+- **Intermediate**: [Features](features.md) → [Camera Support](camera_support.md)
+- **Advanced**: [Processor Plugins](PLUGIN_SYSTEM.md) → Implementation details
+
+---
+
+*Last updated: 2025-10-24*
diff --git a/docs/aravis_backend.md b/docs/aravis_backend.md
new file mode 100644
index 0000000..67024ba
--- /dev/null
+++ b/docs/aravis_backend.md
@@ -0,0 +1,202 @@
+# Aravis Backend
+
+The Aravis backend provides support for GenICam-compatible cameras using the [Aravis](https://github.com/AravisProject/aravis) library.
+
+## Features
+
+- Support for GenICam/GigE Vision cameras
+- Automatic device detection with `get_device_count()`
+- Configurable exposure time and gain
+- Support for various pixel formats (Mono8, Mono12, Mono16, RGB8, BGR8)
+- Efficient streaming with configurable buffer count
+- Timeout handling for robust operation
+
+## Installation
+
+### Linux (Ubuntu/Debian)
+```bash
+sudo apt-get install gir1.2-aravis-0.8 python3-gi
+```
+
+### Linux (Fedora)
+```bash
+sudo dnf install aravis python3-gobject
+```
+
+### Windows
+Aravis support on Windows requires building from source or using WSL. For native Windows support, consider using the GenTL backend instead.
+
+### macOS
+```bash
+brew install aravis
+pip install pygobject
+```
+
+## Configuration
+
+### Basic Configuration
+
+Select "aravis" as the backend in the GUI or in your configuration file:
+
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0,
+ "fps": 30.0,
+ "exposure": 10000,
+ "gain": 5.0
+ }
+}
+```
+
+### Advanced Properties
+
+You can configure additional Aravis-specific properties via the `properties` dictionary:
+
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0,
+ "fps": 30.0,
+ "exposure": 10000,
+ "gain": 5.0,
+ "properties": {
+ "camera_id": "MyCamera-12345",
+ "pixel_format": "Mono8",
+ "timeout": 2000000,
+ "n_buffers": 10
+ }
+ }
+}
+```
+
+#### Available Properties
+
+| Property | Type | Default | Description |
+|----------|------|---------|-------------|
+| `camera_id` | string | None | Specific camera ID to open (overrides index) |
+| `pixel_format` | string | "Mono8" | Pixel format: Mono8, Mono12, Mono16, RGB8, BGR8 |
+| `timeout` | int | 2000000 | Frame timeout in microseconds (2 seconds) |
+| `n_buffers` | int | 10 | Number of buffers in the acquisition stream |
+
+### Exposure and Gain
+
+The Aravis backend supports exposure time (in microseconds) and gain control:
+
+- **Exposure**: Set via the GUI exposure field or `settings.exposure` (0 = auto, >0 = manual in μs)
+- **Gain**: Set via the GUI gain field or `settings.gain` (0.0 = auto, >0.0 = manual value)
+
+When exposure or gain are set to non-zero values, the backend automatically disables auto-exposure and auto-gain.
+
+## Camera Selection
+
+### By Index
+The default method is to select cameras by index (0, 1, 2, etc.):
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0
+ }
+}
+```
+
+### By Camera ID
+You can also select a specific camera by its ID:
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "properties": {
+ "camera_id": "TheImagingSource-12345678"
+ }
+ }
+}
+```
+
+## Supported Pixel Formats
+
+The backend automatically converts different pixel formats to BGR format for consistency:
+
+- **Mono8**: 8-bit grayscale → BGR
+- **Mono12**: 12-bit grayscale → scaled to 8-bit → BGR
+- **Mono16**: 16-bit grayscale → scaled to 8-bit → BGR
+- **RGB8**: 8-bit RGB → BGR (color conversion)
+- **BGR8**: 8-bit BGR (no conversion needed)
+
+## Performance Tuning
+
+### Buffer Count
+Increase `n_buffers` for high-speed cameras or systems with variable latency:
+```json
+{
+ "properties": {
+ "n_buffers": 20
+ }
+}
+```
+
+### Timeout
+Adjust timeout for slower cameras or network cameras:
+```json
+{
+ "properties": {
+ "timeout": 5000000
+ }
+}
+```
+(5 seconds = 5,000,000 microseconds)
+
+## Troubleshooting
+
+### No cameras detected
+1. Verify Aravis installation: `arv-tool-0.8 -l`
+2. Check camera is powered and connected
+3. Ensure proper network configuration for GigE cameras
+4. Check user permissions for USB cameras
+
+### Timeout errors
+- Increase the `timeout` property
+- Check network bandwidth for GigE cameras
+- Verify camera is properly configured and streaming
+
+### Pixel format errors
+- Check camera's supported pixel formats: `arv-tool-0.8 -n features`
+- Try alternative formats: Mono8, RGB8, etc.
+
+## Comparison with GenTL Backend
+
+| Feature | Aravis | GenTL |
+|---------|--------|-------|
+| Platform | Linux (best), macOS | Windows (best), Linux |
+| Camera Support | GenICam/GigE | GenTL producers |
+| Installation | System packages | Vendor CTI files |
+| Performance | Excellent | Excellent |
+| Auto-detection | Yes | Yes |
+
+## Example: The Imaging Source Camera
+
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0,
+ "fps": 60.0,
+ "exposure": 8000,
+ "gain": 10.0,
+ "properties": {
+ "pixel_format": "Mono8",
+ "n_buffers": 15,
+ "timeout": 3000000
+ }
+ }
+}
+```
+
+## Resources
+
+- [Aravis Project](https://github.com/AravisProject/aravis)
+- [GenICam Standard](https://www.emva.org/standards-technology/genicam/)
+- [Python GObject Documentation](https://pygobject.readthedocs.io/)
diff --git a/docs/camera_support.md b/docs/camera_support.md
index 6e36e22..4d9ba22 100644
--- a/docs/camera_support.md
+++ b/docs/camera_support.md
@@ -1,13 +1,85 @@
## Camera Support
-### Windows
-- **The Imaging Source USB3 Cameras**: via code based on [Windows code samples](https://github.com/TheImagingSource/IC-Imaging-Control-Samples) provided by The Imaging Source. To use The Imaging Source USB3 cameras on Windows, you must first [install their drivers](https://www.theimagingsource.com/support/downloads-for-windows/device-drivers/icwdmuvccamtis/) and [C library](https://www.theimagingsource.com/support/downloads-for-windows/software-development-kits-sdks/tisgrabberdll/).
-- **OpenCV compatible cameras**: OpenCV is installed with DeepLabCut-live-GUI, so webcams or other cameras compatible with OpenCV on Windows require no additional installation.
+DeepLabCut-live-GUI supports multiple camera backends for different platforms and camera types:
-### Linux and NVIDIA Jetson Development Kits
+### Supported Backends
-- **OpenCV compatible cameras**: We provide support for many webcams and industrial cameras using OpenCV via Video4Linux drivers. This includes The Imaging Source USB3 cameras (and others, but untested). OpenCV is installed with DeepLabCut-live-GUI.
-- **Aravis Project compatible USB3Vision and GigE Cameras**: [The Aravis Project](https://github.com/AravisProject/aravis) supports a number of popular industrial cameras used in neuroscience, including The Imaging Source, Point Grey, and Basler cameras. To use Aravis Project drivers, please follow their [installation instructions](https://github.com/AravisProject/aravis#installing-aravis). The Aravis Project drivers are supported on the NVIDIA Jetson platform, but there are known bugs (e.g. [here](https://github.com/AravisProject/aravis/issues/324)).
+1. **OpenCV** - Universal webcam and USB camera support (all platforms)
+2. **GenTL** - Industrial cameras via GenTL producers (Windows, Linux)
+3. **Aravis** - GenICam/GigE Vision cameras (Linux, macOS)
+4. **Basler** - Basler cameras via pypylon (all platforms)
+
+### Backend Selection
+
+You can select the backend in the GUI from the "Backend" dropdown, or in your configuration file:
+
+```json
+{
+ "camera": {
+ "backend": "aravis",
+ "index": 0,
+ "fps": 30.0
+ }
+}
+```
+
+### Platform-Specific Recommendations
+
+#### Windows
+- **OpenCV compatible cameras**: Best for webcams and simple USB cameras. OpenCV is installed with DeepLabCut-live-GUI.
+- **GenTL backend**: Recommended for industrial cameras (The Imaging Source, Basler, etc.) via vendor-provided CTI files.
+- **Basler cameras**: Can use either GenTL or pypylon backend.
+
+#### Linux
+- **OpenCV compatible cameras**: Good for webcams via Video4Linux drivers. Installed with DeepLabCut-live-GUI.
+- **Aravis backend**: **Recommended** for GenICam/GigE Vision industrial cameras (The Imaging Source, Basler, Point Grey, etc.)
+ - Easy installation via system package manager
+ - Better Linux support than GenTL
+ - See [Aravis Backend Documentation](aravis_backend.md)
+- **GenTL backend**: Alternative for industrial cameras if vendor provides Linux CTI files.
+
+#### macOS
+- **OpenCV compatible cameras**: For webcams and compatible USB cameras.
+- **Aravis backend**: For GenICam/GigE Vision cameras (requires Homebrew installation).
+
+#### NVIDIA Jetson
+- **OpenCV compatible cameras**: Standard V4L2 camera support.
+- **Aravis backend**: Supported but may have platform-specific bugs. See [Aravis issues](https://github.com/AravisProject/aravis/issues/324).
+
+### Quick Installation Guide
+
+#### Aravis (Linux/Ubuntu)
+```bash
+sudo apt-get install gir1.2-aravis-0.8 python3-gi
+```
+
+#### Aravis (macOS)
+```bash
+brew install aravis
+pip install pygobject
+```
+
+#### GenTL (Windows)
+Install vendor-provided camera drivers and SDK. CTI files are typically in:
+- `C:\Program Files\The Imaging Source Europe GmbH\IC4 GenTL Driver\bin\`
+
+### Backend Comparison
+
+| Feature | OpenCV | GenTL | Aravis | Basler (pypylon) |
+|---------|--------|-------|--------|------------------|
+| Ease of use | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐ |
+| Auto-detection | Basic | Yes | Yes | Yes |
+| Exposure control | Limited | Yes | Yes | Yes |
+| Gain control | Limited | Yes | Yes | Yes |
+| Windows | ✅ | ✅ | ❌ | ✅ |
+| Linux | ✅ | ✅ | ✅ | ✅ |
+| macOS | ✅ | ❌ | ✅ | ✅ |
+
+### Detailed Backend Documentation
+
+- [Aravis Backend](aravis_backend.md) - GenICam/GigE cameras on Linux/macOS
+- GenTL Backend - Industrial cameras via vendor CTI files
+- OpenCV Backend - Universal webcam support
### Contributing New Camera Types
diff --git a/docs/features.md b/docs/features.md
new file mode 100644
index 0000000..5fd535d
--- /dev/null
+++ b/docs/features.md
@@ -0,0 +1,653 @@
+# DeepLabCut-live-GUI Features
+
+## Table of Contents
+
+- [Camera Control](#camera-control)
+- [Real-Time Pose Estimation](#real-time-pose-estimation)
+- [Video Recording](#video-recording)
+- [Configuration Management](#configuration-management)
+- [Processor System](#processor-system)
+- [User Interface](#user-interface)
+- [Performance Monitoring](#performance-monitoring)
+- [Advanced Features](#advanced-features)
+
+---
+
+## Camera Control
+
+### Multi-Backend Support
+
+The GUI supports four different camera backends, each optimized for different use cases:
+
+#### OpenCV Backend
+- **Platform**: Windows, Linux, macOS
+- **Best For**: Webcams, simple USB cameras
+- **Installation**: Built-in with OpenCV
+- **Limitations**: Limited exposure/gain control
+
+#### GenTL Backend (Harvesters)
+- **Platform**: Windows, Linux
+- **Best For**: Industrial cameras with GenTL producers
+- **Installation**: Requires vendor CTI files
+- **Features**: Full camera control, smart device detection
+
+#### Aravis Backend
+- **Platform**: Linux (best), macOS
+- **Best For**: GenICam/GigE Vision cameras
+- **Installation**: System packages (`gir1.2-aravis-0.8`)
+- **Features**: Excellent Linux support, native GigE
+
+#### Basler Backend (pypylon)
+- **Platform**: Windows, Linux, macOS
+- **Best For**: Basler cameras specifically
+- **Installation**: Pylon SDK + pypylon
+- **Features**: Vendor-specific optimizations
+
+### Camera Settings
+
+#### Frame Rate Control
+- Range: 1-240 FPS (hardware dependent)
+- Real-time FPS monitoring
+- Automatic camera validation
+
+#### Exposure Control
+- Auto mode (value = 0)
+- Manual mode (microseconds)
+- Range: 0-1,000,000 μs
+- Real-time adjustment (backend dependent)
+
+#### Gain Control
+- Auto mode (value = 0.0)
+- Manual mode (gain value)
+- Range: 0.0-100.0
+- Useful for low-light conditions
+
+#### Region of Interest (ROI) Cropping
+- Define crop region: (x0, y0, x1, y1)
+- Applied before recording and inference
+- Reduces processing load
+- Maintains aspect ratio
+
+#### Image Rotation
+- 0°, 90°, 180°, 270° rotation
+- Applied to all outputs
+- Useful for mounted cameras
+
+### Smart Camera Detection
+
+The GUI intelligently detects available cameras:
+
+1. **Backend-Specific**: Each backend reports available cameras
+2. **No Blind Probing**: GenTL and Aravis query actual device count
+3. **Fast Refresh**: Only check connected devices
+4. **Detailed Labels**: Shows vendor, model, serial number
+
+Example detection output:
+```
+[CameraDetection] Available cameras for backend 'gentl':
+ ['0:DMK 37BUX287 (26320523)', '1:Basler acA1920 (40123456)']
+```
+
+---
+
+## Real-Time Pose Estimation
+
+### DLCLive Integration
+
+#### Model Support
+- **TensorFlow (Base)**: Original DeepLabCut models
+- **PyTorch**: PyTorch-exported models
+- Model selection via dropdown
+- Automatic model validation
+
+#### Inference Pipeline
+1. **Frame Acquisition**: Camera thread → Queue
+2. **Preprocessing**: Crop, resize (optional)
+3. **Inference**: DLCLive model processing
+4. **Pose Output**: (x, y) coordinates per keypoint
+5. **Visualization**: Optional overlay on video
+
+#### Performance Metrics
+- **Inference FPS**: Actual processing rate
+- **Latency**: Time from capture to pose output
+ - Last latency (ms)
+ - Average latency (ms)
+- **Queue Status**: Frame buffer depth
+- **Dropped Frames**: Count of skipped frames
+
+### Pose Visualization
+
+#### Overlay Options
+- **Toggle**: "Display pose predictions" checkbox
+- **Keypoint Markers**: Green circles at (x, y) positions
+- **Real-Time Update**: Synchronized with video feed
+- **No Performance Impact**: Rendering optimized
+
+#### Bounding Box Visualization
+- **Purpose**: Visual ROI definition
+- **Configuration**: (x0, y0, x1, y1) coordinates
+- **Color**: Red rectangle overlay
+- **Use Cases**:
+ - Crop region preview
+ - Analysis area marking
+ - Multi-region tracking
+
+### Initialization Feedback
+
+Visual indicators during model loading:
+1. **"Initializing DLCLive!"** - Blue button during load
+2. **"DLCLive running!"** - Green button when ready
+3. Status bar updates with progress
+
+---
+
+## Video Recording
+
+### Recording Capabilities
+
+#### Hardware-Accelerated Encoding
+- **NVENC (NVIDIA)**: GPU-accelerated H.264/H.265
+ - Codecs: `h264_nvenc`, `hevc_nvenc`
+ - 10x faster than software encoding
+ - Minimal CPU usage
+- **Software Encoding**: CPU-based fallback
+ - Codecs: `libx264`, `libx265`
+ - Universal compatibility
+
+#### Container Formats
+- **MP4**: Most compatible, web-ready
+- **AVI**: Legacy support
+- **MOV**: Apple ecosystem
+
+#### Quality Control
+- **CRF (Constant Rate Factor)**: 0-51
+ - 0 = Lossless (huge files)
+ - 23 = Default (good quality)
+ - 28 = High compression
+ - 51 = Lowest quality
+- **Presets**: ultrafast, fast, medium, slow
+
+### Recording Features
+
+#### Timestamp Synchronization
+- Frame-accurate timestamps
+- Microsecond precision
+- Synchronized with pose data
+- Stored in separate files
+
+#### Performance Monitoring
+- **Write FPS**: Actual encoding rate
+- **Queue Size**: Buffer depth (~ms)
+- **Latency**: Encoding delay
+- **Frames Written/Enqueued**: Progress tracking
+- **Dropped Frames**: Quality indicator
+
+#### Buffer Management
+- Configurable queue size
+- Automatic overflow handling
+- Warning on frame drops
+- Backpressure indication
+
+### Auto-Recording Feature
+
+Processor-triggered recording:
+
+1. **Enable**: Check "Auto-record video on processor command"
+2. **Processor Control**: Custom processor sets recording flag
+3. **Automatic Start**: GUI starts recording when flag set
+4. **Session Naming**: Uses processor-defined session name
+5. **Automatic Stop**: GUI stops when flag cleared
+
+**Use Cases**:
+- Event-triggered recording
+- Trial-based experiments
+- Conditional data capture
+- Remote control via socket
+
+---
+
+## Configuration Management
+
+### Configuration File Structure
+
+Single JSON file contains all settings:
+
+```json
+{
+ "camera": { ... },
+ "dlc": { ... },
+ "recording": { ... },
+ "bbox": { ... }
+}
+```
+
+### Features
+
+#### Save/Load Operations
+- **Load**: File → Load configuration (Ctrl+O)
+- **Save**: File → Save configuration (Ctrl+S)
+- **Save As**: File → Save configuration as (Ctrl+Shift+S)
+- **Auto-sync**: GUI fields update from file
+
+#### Multiple Configurations
+- Switch between experiments quickly
+- Per-animal configurations
+- Environment-specific settings
+- Backup and version control
+
+#### Validation
+- Type checking on load
+- Default values for missing fields
+- Error messages for invalid entries
+- Safe fallback to defaults
+
+### Configuration Sections
+
+#### Camera Settings (`camera`)
+```json
+{
+ "name": "Camera 0",
+ "index": 0,
+ "fps": 60.0,
+ "backend": "gentl",
+ "exposure": 10000,
+ "gain": 5.0,
+ "crop_x0": 0,
+ "crop_y0": 0,
+ "crop_x1": 0,
+ "crop_y1": 0,
+ "max_devices": 3,
+ "properties": {}
+}
+```
+
+#### DLC Settings (`dlc`)
+```json
+{
+ "model_path": "/path/to/model",
+ "model_type": "base",
+ "additional_options": {
+ "resize": 0.5,
+ "processor": "cpu",
+ "pcutoff": 0.6
+ }
+}
+```
+
+#### Recording Settings (`recording`)
+```json
+{
+ "enabled": true,
+ "directory": "~/Videos/dlc",
+ "filename": "session.mp4",
+ "container": "mp4",
+ "codec": "h264_nvenc",
+ "crf": 23
+}
+```
+
+#### Bounding Box Settings (`bbox`)
+```json
+{
+ "enabled": false,
+ "x0": 0,
+ "y0": 0,
+ "x1": 200,
+ "y1": 100
+}
+```
+
+---
+
+## Processor System
+
+### Plugin Architecture
+
+Custom pose processors for real-time analysis and control.
+
+#### Processor Interface
+
+```python
+class MyProcessor:
+ """Custom processor example."""
+
+ def process(self, pose, timestamp):
+ """Process pose data in real-time.
+
+ Args:
+ pose: numpy array (n_keypoints, 3) - x, y, likelihood
+ timestamp: float - frame timestamp
+ """
+ # Extract keypoint positions
+ nose_x, nose_y = pose[0, :2]
+
+ # Custom logic
+ if nose_x > 320:
+ self.trigger_event()
+
+ # Return results (optional)
+ return {"position": (nose_x, nose_y)}
+```
+
+#### Loading Processors
+
+1. Place processor file in `dlclivegui/processors/`
+2. Click "Refresh" in processor dropdown
+3. Select processor from list
+4. Start inference to activate
+
+#### Built-in Processors
+
+**Socket Processor** (`dlc_processor_socket.py`):
+- TCP socket server for remote control
+- Commands: `START_RECORDING`, `STOP_RECORDING`
+- Session management
+- Multi-client support
+
+### Auto-Recording Integration
+
+Processors can control recording:
+
+```python
+class RecordingProcessor:
+ def __init__(self):
+ self._vid_recording = False
+ self.session_name = "default"
+
+ @property
+ def video_recording(self):
+ return self._vid_recording
+
+ def start_recording(self, session):
+ self.session_name = session
+ self._vid_recording = True
+
+ def stop_recording(self):
+ self._vid_recording = False
+```
+
+The GUI monitors `video_recording` property and automatically starts/stops recording.
+
+---
+
+## User Interface
+
+### Layout
+
+#### Control Panel (Left)
+- **Camera Settings**: Backend, index, FPS, exposure, gain, crop
+- **DLC Settings**: Model path, type, processor, options
+- **Recording Settings**: Path, filename, codec, quality
+- **Bounding Box**: Visualization controls
+
+#### Video Display (Right)
+- Live camera feed
+- Pose overlay (optional)
+- Bounding box overlay (optional)
+- Auto-scaling to window size
+
+#### Status Bar (Bottom)
+- Current operation status
+- Error messages
+- Success confirmations
+
+### Control Groups
+
+#### Camera Controls
+- Backend selection dropdown
+- Camera index/refresh
+- FPS, exposure, gain spinboxes
+- Crop coordinates
+- Rotation selector
+- **Start/Stop Preview** buttons
+
+#### DLC Controls
+- Model path browser
+- Model type selector
+- Processor folder/selection
+- Additional options (JSON)
+- **Start/Stop Inference** buttons
+- "Display pose predictions" checkbox
+- "Auto-record" checkbox
+- Processor status display
+
+#### Recording Controls
+- Output directory browser
+- Filename input
+- Container/codec selectors
+- CRF quality slider
+- **Start/Stop Recording** buttons
+
+### Visual Feedback
+
+#### Button States
+- **Disabled**: Gray, not clickable
+- **Enabled**: Default color, clickable
+- **Active**:
+ - Preview running: Stop button enabled
+ - Inference initializing: Blue "Initializing DLCLive!"
+ - Inference ready: Green "DLCLive running!"
+
+#### Status Indicators
+- Camera FPS (last 5 seconds)
+- DLC performance metrics
+- Recording statistics
+- Processor connection status
+
+---
+
+## Performance Monitoring
+
+### Real-Time Metrics
+
+#### Camera Metrics
+- **Throughput**: FPS over last 5 seconds
+- **Formula**: `(frame_count - 1) / time_elapsed`
+- **Display**: "45.2 fps (last 5 s)"
+
+#### DLC Metrics
+- **Inference FPS**: Poses processed per second
+- **Latency**:
+ - Last frame latency (ms)
+ - Average latency over session (ms)
+- **Queue**: Number of frames waiting
+- **Dropped**: Frames skipped due to queue full
+- **Format**: "150/152 frames | inference 42.1 fps | latency 23.5 ms (avg 24.1 ms) | queue 2 | dropped 2"
+
+#### Recording Metrics
+- **Write FPS**: Encoding rate
+- **Frames**: Written/Enqueued ratio
+- **Latency**: Encoding delay (ms)
+- **Buffer**: Queue size (~milliseconds)
+- **Dropped**: Encoding failures
+- **Format**: "1500/1502 frames | write 59.8 fps | latency 12.3 ms (avg 12.5 ms) | queue 5 (~83 ms) | dropped 2"
+
+### Performance Optimization
+
+#### Automatic Adjustments
+- Frame display throttling (25 Hz max)
+- Queue backpressure handling
+- Automatic resolution detection
+
+#### User Adjustments
+- Reduce camera FPS
+- Enable ROI cropping
+- Use hardware encoding
+- Increase CRF value
+- Disable pose visualization
+- Adjust buffer counts
+
+---
+
+## Advanced Features
+
+### Frame Synchronization
+
+All components share frame timestamps:
+- Camera controller generates timestamps
+- DLC processor preserves timestamps
+- Video recorder stores timestamps
+- Enables post-hoc alignment
+
+### Error Recovery
+
+#### Camera Connection Loss
+- Automatic detection via frame grab failure
+- User notification
+- Clean resource cleanup
+- Restart capability
+
+#### Recording Errors
+- Frame size mismatch detection
+- Automatic recovery with new settings
+- Warning display
+- No data loss
+
+### Thread Safety
+
+Multi-threaded architecture:
+- **Main Thread**: GUI event loop
+- **Camera Thread**: Frame acquisition
+- **DLC Thread**: Pose inference
+- **Recording Thread**: Video encoding
+
+Qt signals/slots ensure thread-safe communication.
+
+### Resource Management
+
+#### Automatic Cleanup
+- Camera release on stop/error
+- DLC model unload on stop
+- Recording finalization
+- Thread termination
+
+#### Memory Management
+- Bounded queues prevent memory leaks
+- Frame copy-on-write
+- Efficient numpy array handling
+
+### Extensibility
+
+#### Custom Backends
+Implement `CameraBackend` abstract class:
+```python
+class MyBackend(CameraBackend):
+ def open(self): ...
+ def read(self) -> Tuple[np.ndarray, float]: ...
+ def close(self): ...
+
+ @classmethod
+ def get_device_count(cls) -> int: ...
+```
+
+Register in `factory.py`:
+```python
+_BACKENDS = {
+ "mybackend": ("module.path", "MyBackend")
+}
+```
+
+#### Custom Processors
+Place in `processors/` directory:
+```python
+class MyProcessor:
+ def __init__(self, **kwargs):
+ # Initialize
+ pass
+
+ def process(self, pose, timestamp):
+ # Process pose
+ pass
+```
+
+### Debugging Features
+
+#### Logging
+- Console output for errors
+- Frame acquisition logging
+- Performance warnings
+- Connection status
+
+#### Development Mode
+- Syntax validation: `python -m compileall dlclivegui`
+- Type checking: `mypy dlclivegui`
+- Test files included
+
+---
+
+## Use Case Examples
+
+### High-Speed Behavior Tracking
+
+**Setup**:
+- Camera: GenTL industrial camera @ 120 FPS
+- Codec: h264_nvenc (GPU encoding)
+- Crop: Region of interest only
+- DLC: PyTorch model on GPU
+
+**Settings**:
+```json
+{
+ "camera": {"fps": 120, "crop_x0": 200, "crop_y0": 100, "crop_x1": 800, "crop_y1": 600},
+ "recording": {"codec": "h264_nvenc", "crf": 28},
+ "dlc": {"additional_options": {"processor": "gpu", "resize": 0.5}}
+}
+```
+
+### Event-Triggered Recording
+
+**Setup**:
+- Processor: Socket processor with auto-record
+- Trigger: Remote computer sends START/STOP commands
+- Session naming: Unique per trial
+
+**Workflow**:
+1. Enable "Auto-record video on processor command"
+2. Start preview and inference
+3. Remote system connects via socket
+4. Sends `START_RECORDING:trial_001` → recording starts
+5. Sends `STOP_RECORDING` → recording stops
+6. Files saved as `trial_001.mp4`
+
+### Multi-Camera Synchronization
+
+**Setup**:
+- Multiple GUI instances
+- Shared trigger signal
+- Synchronized filenames
+
+**Configuration**:
+Each instance with different camera index but same settings template.
+
+---
+
+## Keyboard Shortcuts
+
+- **Ctrl+O**: Load configuration
+- **Ctrl+S**: Save configuration
+- **Ctrl+Shift+S**: Save configuration as
+- **Ctrl+Q**: Quit application
+
+---
+
+## Platform-Specific Notes
+
+### Windows
+- Best GenTL support (vendor CTI files)
+- NVENC highly recommended
+- DirectShow backend for webcams
+
+### Linux
+- Best Aravis support (native GigE)
+- V4L2 backend for webcams
+- NVENC available with proprietary drivers
+
+### macOS
+- Limited industrial camera support
+- Aravis via Homebrew
+- Software encoding recommended
+
+### NVIDIA Jetson
+- Optimized for edge deployment
+- Hardware encoding available
+- Some Aravis compatibility issues
diff --git a/docs/install.md b/docs/install.md
index fa69ab4..24ef4f4 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -22,4 +22,4 @@ pip install deeplabcut-live-gui
First, please refer to our complete instructions for [installing DeepLabCut-Live! on a NVIDIA Jetson Development Kit](https://github.com/DeepLabCut/DeepLabCut-live/blob/master/docs/install_jetson.md).
-Next, install the DeepLabCut-live-GUI: `pip install deeplabcut-live-gui`.
\ No newline at end of file
+Next, install the DeepLabCut-live-GUI: `pip install deeplabcut-live-gui`.
diff --git a/docs/timestamp_format.md b/docs/timestamp_format.md
new file mode 100644
index 0000000..ac5e5a7
--- /dev/null
+++ b/docs/timestamp_format.md
@@ -0,0 +1,79 @@
+# Video Frame Timestamp Format
+
+When recording video, the application automatically saves frame timestamps to a JSON file alongside the video file.
+
+## File Naming
+
+For a video file named `recording_2025-10-23_143052.mp4`, the timestamp file will be:
+```
+recording_2025-10-23_143052.mp4_timestamps.json
+```
+
+## JSON Structure
+
+```json
+{
+ "video_file": "recording_2025-10-23_143052.mp4",
+ "num_frames": 1500,
+ "timestamps": [
+ 1729693852.123456,
+ 1729693852.156789,
+ 1729693852.190123,
+ ...
+ ],
+ "start_time": 1729693852.123456,
+ "end_time": 1729693902.123456,
+ "duration_seconds": 50.0
+}
+```
+
+## Fields
+
+- **video_file**: Name of the associated video file
+- **num_frames**: Total number of frames recorded
+- **timestamps**: Array of Unix timestamps (seconds since epoch with microsecond precision) for each frame
+- **start_time**: Timestamp of the first frame
+- **end_time**: Timestamp of the last frame
+- **duration_seconds**: Total recording duration in seconds
+
+## Usage
+
+The timestamps correspond to the exact time each frame was captured by the camera (from `FrameData.timestamp`). This allows precise synchronization with:
+
+- DLC pose estimation results
+- External sensors or triggers
+- Other data streams recorded during the same session
+
+## Example: Loading Timestamps in Python
+
+```python
+import json
+from datetime import datetime
+
+# Load timestamps
+with open('recording_2025-10-23_143052.mp4_timestamps.json', 'r') as f:
+ data = json.load(f)
+
+print(f"Video: {data['video_file']}")
+print(f"Total frames: {data['num_frames']}")
+print(f"Duration: {data['duration_seconds']:.2f} seconds")
+
+# Convert first timestamp to human-readable format
+start_dt = datetime.fromtimestamp(data['start_time'])
+print(f"Recording started: {start_dt.isoformat()}")
+
+# Calculate average frame rate
+avg_fps = data['num_frames'] / data['duration_seconds']
+print(f"Average FPS: {avg_fps:.2f}")
+
+# Access individual frame timestamps
+for frame_idx, timestamp in enumerate(data['timestamps']):
+ print(f"Frame {frame_idx}: {timestamp}")
+```
+
+## Notes
+
+- Timestamps use `time.time()` which returns Unix epoch time with high precision
+- Frame timestamps are captured when frames arrive from the camera, before any processing
+- If frames are dropped due to queue overflow, those frames will not have timestamps in the array
+- The timestamp array length should match the number of frames in the video file
diff --git a/docs/user_guide.md b/docs/user_guide.md
new file mode 100644
index 0000000..289bfb8
--- /dev/null
+++ b/docs/user_guide.md
@@ -0,0 +1,633 @@
+# DeepLabCut-live-GUI User Guide
+
+Complete walkthrough for using the DeepLabCut-live-GUI application.
+
+## Table of Contents
+
+1. [Getting Started](#getting-started)
+2. [Camera Setup](#camera-setup)
+3. [DLCLive Configuration](#dlclive-configuration)
+4. [Recording Videos](#recording-videos)
+5. [Working with Configurations](#working-with-configurations)
+6. [Common Workflows](#common-workflows)
+7. [Tips and Best Practices](#tips-and-best-practices)
+
+---
+
+## Getting Started
+
+### First Launch
+
+1. Open a terminal/command prompt
+2. Run the application:
+ ```bash
+ dlclivegui
+ ```
+3. The main window will appear with three control panels and a video display area
+
+### Interface Overview
+
+```
+┌─────────────────────────────────────────────────────┐
+│ File Help │
+├─────────────┬───────────────────────────────────────┤
+│ Camera │ │
+│ Settings │ │
+│ │ │
+│ ─────────── │ Video Display │
+│ DLCLive │ │
+│ Settings │ │
+│ │ │
+│ ─────────── │ │
+│ Recording │ │
+│ Settings │ │
+│ │ │
+│ ─────────── │ │
+│ Bounding │ │
+│ Box │ │
+│ │ │
+│ ─────────── │ │
+│ [Preview] │ │
+│ [Stop] │ │
+└─────────────┴───────────────────────────────────────┘
+│ Status: Ready │
+└─────────────────────────────────────────────────────┘
+```
+
+---
+
+## Camera Setup
+
+### Step 1: Select Camera Backend
+
+The **Backend** dropdown shows available camera drivers:
+
+| Backend | When to Use |
+|---------|-------------|
+| **opencv** | Webcams, USB cameras (universal) |
+| **gentl** | Industrial cameras (Windows/Linux) |
+| **aravis** | GenICam/GigE cameras (Linux/macOS) |
+| **basler** | Basler cameras specifically |
+
+**Note**: Unavailable backends appear grayed out. Install required drivers to enable them.
+
+### Step 2: Select Camera
+
+1. Click **Refresh** next to the camera dropdown
+2. Wait for camera detection (1-3 seconds)
+3. Select your camera from the dropdown
+
+The list shows camera details:
+```
+0:DMK 37BUX287 (26320523)
+│ │ └─ Serial Number
+│ └─ Model Name
+└─ Index
+```
+
+### Step 3: Configure Camera Parameters
+
+#### Frame Rate
+- **Range**: 1-240 FPS (hardware dependent)
+- **Recommendation**: Start with 30 FPS, increase as needed
+- **Note**: Higher FPS = more processing load
+
+#### Exposure Time
+- **Auto**: Set to 0 (default)
+- **Manual**: Microseconds (e.g., 10000 = 10ms)
+- **Tips**:
+ - Shorter exposure = less motion blur
+ - Longer exposure = better low-light performance
+ - Typical range: 5,000-30,000 μs
+
+#### Gain
+- **Auto**: Set to 0.0 (default)
+- **Manual**: 0.0-100.0
+- **Tips**:
+ - Higher gain = brighter image but more noise
+ - Start low (5-10) and increase if needed
+ - Auto mode works well for most cases
+
+#### Cropping (Optional)
+Reduce frame size for faster processing:
+
+1. Set crop region: (x0, y0, x1, y1)
+ - x0, y0: Top-left corner
+ - x1, y1: Bottom-right corner
+2. Use Bounding Box visualization to preview
+3. Set all to 0 to disable cropping
+
+**Example**: Crop to center 640x480 region of 1280x720 camera:
+```
+x0: 320
+y0: 120
+x1: 960
+y1: 600
+```
+
+#### Rotation
+Select if camera is mounted at an angle:
+- 0° (default)
+- 90° (rotated right)
+- 180° (upside down)
+- 270° (rotated left)
+
+### Step 4: Start Camera Preview
+
+1. Click **Start Preview**
+2. Video feed should appear in the display area
+3. Check the **Throughput** metric below camera settings
+4. Verify frame rate matches expected value
+
+**Troubleshooting**:
+- **No preview**: Check camera connection and permissions
+- **Low FPS**: Reduce resolution or increase exposure time
+- **Black screen**: Check exposure settings
+- **Distorted image**: Verify backend compatibility
+
+---
+
+## DLCLive Configuration
+
+### Prerequisites
+
+1. Exported DLCLive model (see DLC documentation)
+2. DeepLabCut-live installed (`pip install deeplabcut-live`)
+3. Camera preview running
+
+### Step 1: Select Model
+
+1. Click **Browse** next to "Model directory"
+2. Navigate to your exported DLCLive model folder
+3. Select the folder containing:
+ - `pose_cfg.yaml`
+ - Model weights (`.pb`, `.pth`, etc.)
+
+### Step 2: Choose Model Type
+
+Select from dropdown:
+- **Base (TensorFlow)**: Standard DLC models
+- **PyTorch**: PyTorch-based models (requires PyTorch)
+
+### Step 3: Configure Options (Optional)
+
+Click in "Additional options" field and enter JSON:
+
+```json
+{
+ "processor": "gpu",
+ "resize": 0.5,
+ "pcutoff": 0.6
+}
+```
+
+**Common options**:
+- `processor`: "cpu" or "gpu"
+- `resize`: Scale factor (0.5 = half size)
+- `pcutoff`: Likelihood threshold
+- `cropping`: Crop before inference
+
+### Step 4: Select Processor (Optional)
+
+If using custom pose processors:
+
+1. Click **Browse** next to "Processor folder" (or use default)
+2. Click **Refresh** to scan for processors
+3. Select processor from dropdown
+4. Processor will activate when inference starts
+
+### Step 5: Start Inference
+
+1. Ensure camera preview is running
+2. Click **Start pose inference**
+3. Button changes to "Initializing DLCLive!" (blue)
+4. Wait for model loading (5-30 seconds)
+5. Button changes to "DLCLive running!" (green)
+6. Check **Performance** metrics
+
+**Performance Metrics**:
+```
+150/152 frames | inference 42.1 fps | latency 23.5 ms (avg 24.1 ms) | queue 2 | dropped 2
+```
+- **150/152**: Processed/Total frames
+- **inference 42.1 fps**: Processing rate
+- **latency 23.5 ms**: Current processing delay
+- **queue 2**: Frames waiting
+- **dropped 2**: Skipped frames (due to full queue)
+
+### Step 6: Enable Visualization (Optional)
+
+Check **"Display pose predictions"** to overlay keypoints on video.
+
+- Keypoints appear as green circles
+- Updates in real-time with video
+- Can be toggled during inference
+
+---
+
+## Recording Videos
+
+### Basic Recording
+
+1. **Configure output path**:
+ - Click **Browse** next to "Output directory"
+ - Select or create destination folder
+
+2. **Set filename**:
+ - Enter base filename (e.g., "session_001")
+ - Extension added automatically based on container
+
+3. **Select format**:
+ - **Container**: mp4 (recommended), avi, mov
+ - **Codec**:
+ - `h264_nvenc` (NVIDIA GPU - fastest)
+ - `libx264` (CPU - universal)
+ - `hevc_nvenc` (NVIDIA H.265)
+
+4. **Set quality** (CRF slider):
+ - 0-17: Very high quality, large files
+ - 18-23: High quality (recommended)
+ - 24-28: Medium quality, smaller files
+ - 29-51: Lower quality, smallest files
+
+5. **Start recording**:
+ - Ensure camera preview is running
+ - Click **Start recording**
+ - **Stop recording** button becomes enabled
+
+6. **Monitor performance**:
+ - Check "Performance" metrics
+ - Watch for dropped frames
+ - Verify write FPS matches camera FPS
+
+### Advanced Recording Options
+
+#### High-Speed Recording (60+ FPS)
+
+**Settings**:
+- Codec: `h264_nvenc` (requires NVIDIA GPU)
+- CRF: 28 (higher compression)
+- Crop region: Reduce frame size
+- Close other applications
+
+#### High-Quality Recording
+
+**Settings**:
+- Codec: `libx264` or `h264_nvenc`
+- CRF: 18-20
+- Full resolution
+- Sufficient disk space
+
+#### Long Duration Recording
+
+**Tips**:
+- Use CRF 23-25 to balance quality/size
+- Monitor disk space
+- Consider splitting into multiple files
+- Use fast SSD storage
+
+### Auto-Recording
+
+Enable automatic recording triggered by processor events:
+
+1. **Select a processor** that supports auto-recording
+2. **Enable**: Check "Auto-record video on processor command"
+3. **Start inference**: Processor will control recording
+4. **Session management**: Files named by processor
+
+**Use cases**:
+- Trial-based experiments
+- Event-triggered recording
+- Remote control via socket processor
+- Conditional data capture
+
+---
+
+## Working with Configurations
+
+### Saving Current Settings
+
+**Save** (overwrites existing file):
+1. File → Save configuration (or Ctrl+S)
+2. If no file loaded, prompts for location
+
+**Save As** (create new file):
+1. File → Save configuration as… (or Ctrl+Shift+S)
+2. Choose location and filename
+3. Enter name (e.g., `mouse_experiment.json`)
+4. Click Save
+
+### Loading Saved Settings
+
+1. File → Load configuration… (or Ctrl+O)
+2. Navigate to configuration file
+3. Select `.json` file
+4. Click Open
+5. All GUI fields update automatically
+
+### Managing Multiple Configurations
+
+**Recommended structure**:
+```
+configs/
+├── default.json # Base settings
+├── mouse_arena1.json # Arena-specific
+├── mouse_arena2.json
+├── rat_setup.json
+└── high_speed.json # Performance-specific
+```
+
+**Workflow**:
+1. Create base configuration with common settings
+2. Save variants for different:
+ - Animals/subjects
+ - Experimental setups
+ - Camera positions
+ - Recording quality levels
+
+### Configuration Templates
+
+#### Webcam + CPU Processing
+```json
+{
+ "camera": {
+ "backend": "opencv",
+ "index": 0,
+ "fps": 30.0
+ },
+ "dlc": {
+ "model_type": "base",
+ "additional_options": {"processor": "cpu"}
+ },
+ "recording": {
+ "codec": "libx264",
+ "crf": 23
+ }
+}
+```
+
+#### Industrial Camera + GPU
+```json
+{
+ "camera": {
+ "backend": "gentl",
+ "index": 0,
+ "fps": 60.0,
+ "exposure": 10000,
+ "gain": 8.0
+ },
+ "dlc": {
+ "model_type": "pytorch",
+ "additional_options": {
+ "processor": "gpu",
+ "resize": 0.5
+ }
+ },
+ "recording": {
+ "codec": "h264_nvenc",
+ "crf": 23
+ }
+}
+```
+
+---
+
+## Common Workflows
+
+### Workflow 1: Simple Webcam Tracking
+
+**Goal**: Track mouse behavior with webcam
+
+1. **Camera Setup**:
+ - Backend: opencv
+ - Camera: Built-in webcam (index 0)
+ - FPS: 30
+
+2. **Start Preview**: Verify mouse is visible
+
+3. **Load DLC Model**: Browse to mouse tracking model
+
+4. **Start Inference**: Enable pose estimation
+
+5. **Verify Tracking**: Enable pose visualization
+
+6. **Record Trial**: Start/stop recording as needed
+
+### Workflow 2: High-Speed Industrial Camera
+
+**Goal**: Track fast movements at 120 FPS
+
+1. **Camera Setup**:
+ - Backend: gentl or aravis
+ - Refresh and select camera
+ - FPS: 120
+ - Exposure: 4000 μs (short exposure)
+ - Crop: Region of interest only
+
+2. **Start Preview**: Check FPS is stable
+
+3. **Configure Recording**:
+ - Codec: h264_nvenc
+ - CRF: 28
+ - Output: Fast SSD
+
+4. **Load DLC Model** (if needed):
+ - PyTorch model
+ - GPU processor
+ - Resize: 0.5 (reduce load)
+
+5. **Start Recording**: Begin data capture
+
+6. **Monitor Performance**: Watch for dropped frames
+
+### Workflow 3: Event-Triggered Recording
+
+**Goal**: Record only during specific events
+
+1. **Camera Setup**: Configure as normal
+
+2. **Processor Setup**:
+ - Select socket processor
+ - Enable "Auto-record video on processor command"
+
+3. **Start Preview**: Camera running
+
+4. **Start Inference**: DLC + processor active
+
+5. **Remote Control**:
+ - Connect to socket (default port 5000)
+ - Send `START_RECORDING:trial_001`
+ - Recording starts automatically
+ - Send `STOP_RECORDING`
+ - Recording stops, file saved
+
+### Workflow 4: Multi-Subject Tracking
+
+**Goal**: Track multiple animals simultaneously
+
+**Option A: Single Camera, Multiple Keypoints**
+1. Use DLC model trained for multiple subjects
+2. Single GUI instance
+3. Processor distinguishes subjects
+
+**Option B: Multiple Cameras**
+1. Launch multiple GUI instances
+2. Each with different camera index
+3. Synchronized configurations
+4. Coordinated filenames
+
+---
+
+## Tips and Best Practices
+
+### Camera Tips
+
+1. **Lighting**:
+ - Consistent, diffuse lighting
+ - Avoid shadows and reflections
+ - IR lighting for night vision
+
+2. **Positioning**:
+ - Stable mount (minimize vibration)
+ - Appropriate angle for markers
+ - Sufficient field of view
+
+3. **Settings**:
+ - Start with auto exposure/gain
+ - Adjust manually if needed
+ - Test different FPS rates
+ - Use cropping to reduce load
+
+### Recording Tips
+
+1. **File Management**:
+ - Use descriptive filenames
+ - Include date/subject/trial info
+ - Organize by experiment/session
+ - Regular backups
+
+2. **Performance**:
+ - Close unnecessary applications
+ - Monitor disk space
+ - Use SSD for high-speed recording
+ - Enable GPU encoding if available
+
+3. **Quality**:
+ - Test CRF values beforehand
+ - Balance quality vs. file size
+ - Consider post-processing needs
+ - Verify recordings occasionally
+
+### DLCLive Tips
+
+1. **Model Selection**:
+ - Use model trained on similar conditions
+ - Test offline before live use
+ - Consider resize for speed
+ - GPU highly recommended
+
+2. **Performance**:
+ - Monitor inference FPS
+ - Check latency values
+ - Watch queue depth
+ - Reduce resolution if needed
+
+3. **Validation**:
+ - Enable visualization initially
+ - Verify tracking quality
+ - Check all keypoints
+ - Test edge cases
+
+### General Best Practices
+
+1. **Configuration Management**:
+ - Save configurations frequently
+ - Version control config files
+ - Document custom settings
+ - Share team configurations
+
+2. **Testing**:
+ - Test setup before experiments
+ - Run trial recordings
+ - Verify all components
+ - Check file outputs
+
+3. **Troubleshooting**:
+ - Check status messages
+ - Monitor performance metrics
+ - Review error dialogs carefully
+ - Restart if issues persist
+
+4. **Data Organization**:
+ - Consistent naming scheme
+ - Separate folders per session
+ - Include metadata files
+ - Regular data validation
+
+---
+
+## Troubleshooting Guide
+
+### Camera Issues
+
+**Problem**: Camera not detected
+- **Solution**: Click Refresh, check connections, verify drivers
+
+**Problem**: Low frame rate
+- **Solution**: Reduce resolution, increase exposure, check CPU usage
+
+**Problem**: Image too dark/bright
+- **Solution**: Adjust exposure and gain settings
+
+### DLCLive Issues
+
+**Problem**: Model fails to load
+- **Solution**: Verify path, check model type, install dependencies
+
+**Problem**: Slow inference
+- **Solution**: Enable GPU, reduce resolution, use resize option
+
+**Problem**: Poor tracking
+- **Solution**: Check lighting, enable visualization, verify model quality
+
+### Recording Issues
+
+**Problem**: Dropped frames
+- **Solution**: Use GPU encoding, increase CRF, reduce FPS
+
+**Problem**: Large file sizes
+- **Solution**: Increase CRF value, use better codec
+
+**Problem**: Recording won't start
+- **Solution**: Check disk space, verify path permissions
+
+---
+
+## Keyboard Reference
+
+| Action | Shortcut |
+|--------|----------|
+| Load configuration | Ctrl+O |
+| Save configuration | Ctrl+S |
+| Save configuration as | Ctrl+Shift+S |
+| Quit application | Ctrl+Q |
+
+---
+
+## Next Steps
+
+- Explore [Features Documentation](features.md) for detailed capabilities
+- Review [Camera Backend Guide](camera_support.md) for advanced setup
+- Check [Processor System](PLUGIN_SYSTEM.md) for custom processing
+- See [Aravis Backend](aravis_backend.md) for Linux industrial cameras
+
+---
+
+## Getting Help
+
+If you encounter issues:
+1. Check status messages in GUI
+2. Review this user guide
+3. Consult technical documentation
+4. Check GitHub issues
+5. Contact support team
From ddcbc419350baec4e755b1ca36fb64a768753a04 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Tue, 28 Oct 2025 15:34:36 +0100
Subject: [PATCH 15/26] update dlc_processor, fix filtering
---
dlclivegui/processors/dlc_processor_socket.py | 58 ++++++++++++++++---
1 file changed, 49 insertions(+), 9 deletions(-)
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index fb6522c..3ab5866 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -1,5 +1,6 @@
import logging
import pickle
+import socket
import time
from collections import deque
from math import acos, atan2, copysign, degrees, pi, sqrt
@@ -7,7 +8,7 @@
from threading import Event, Thread
import numpy as np
-from dlclive import Processor
+from dlclive import Processor # type: ignore
LOG = logging.getLogger("dlc_processor_socket")
LOG.setLevel(logging.INFO)
@@ -42,7 +43,8 @@ def exponential_smoothing(alpha, x, x_prev):
def __call__(self, t, x):
t_e = t - self.t_prev
-
+ if t_e <= 0:
+ return x
a_d = self.smoothing_factor(t_e, self.d_cutoff)
dx = (x - self.x_prev) / t_e
dx_hat = self.exponential_smoothing(a_d, dx, self.dx_prev)
@@ -223,6 +225,31 @@ def _handle_client_message(self, msg):
self._vid_recording.set()
LOG.info("Start video recording command received")
+ elif cmd == "set_filter":
+ # Handle filter enable/disable (subclasses override if they support filtering)
+ use_filter = msg.get("use_filter", False)
+ if hasattr(self, 'use_filter'):
+ self.use_filter = bool(use_filter)
+ # Reset filters to reinitialize with new setting
+ if hasattr(self, 'filters'):
+ self.filters = None
+ LOG.info(f"Filtering {'enabled' if use_filter else 'disabled'}")
+ else:
+ LOG.warning("set_filter command not supported by this processor")
+
+ elif cmd == "set_filter_params":
+ # Handle filter parameter updates (subclasses override if they support filtering)
+ filter_kwargs = msg.get("filter_kwargs", {})
+ if hasattr(self, 'filter_kwargs'):
+ # Update filter parameters
+ self.filter_kwargs.update(filter_kwargs)
+ # Reset filters to reinitialize with new parameters
+ if hasattr(self, 'filters'):
+ self.filters = None
+ LOG.info(f"Filter parameters updated: {filter_kwargs}")
+ else:
+ LOG.warning("set_filter_params command not supported by this processor")
+
def _clear_data_queues(self):
"""Clear all data storage queues. Override in subclasses to clear additional queues."""
self.time_stamp.clear()
@@ -286,17 +313,30 @@ def process(self, pose, **kwargs):
def stop(self):
"""Stop the processor and close all connections."""
+ LOG.info("Stopping processor...")
+
+ # Signal stop to all threads
self._stop.set()
- try:
- self.listener.close()
- except Exception:
- pass
+
+ # Close all client connections first
for c in list(self.conns):
try:
c.close()
except Exception:
pass
self.conns.discard(c)
+
+ # Close the listener socket
+ if hasattr(self, 'listener') and self.listener:
+ try:
+ self.listener.close()
+ except Exception as e:
+ LOG.debug(f"Error closing listener: {e}")
+
+ # Give the OS time to release the socket on Windows
+ # This prevents WinError 10048 when restarting
+ time.sleep(0.1)
+
LOG.info("Processor stopped, all connections closed")
def save(self, file=None):
@@ -306,7 +346,7 @@ def save(self, file=None):
LOG.info(f"Saving data to {file}")
try:
save_dict = self.get_data()
- pickle.dump(save_dict, open(file, "wb"))
+ pickle.dump(save_dict, open(file, "wb"))
save_code = 1
except Exception as e:
LOG.error(f"Save failed: {e}")
@@ -383,7 +423,7 @@ def __init__(
authkey=b"secret password",
use_perf_counter=False,
use_filter=False,
- filter_kwargs=None,
+ filter_kwargs={},
save_original=False,
):
"""
@@ -412,7 +452,7 @@ def __init__(
# Filtering
self.use_filter = use_filter
- self.filter_kwargs = filter_kwargs or {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0}
+ self.filter_kwargs = filter_kwargs
self.filters = None # Will be initialized on first pose
def _clear_data_queues(self):
From e2ff610eca9e918fb2cd9c32ad93e765bcbb8df2 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Wed, 29 Oct 2025 18:33:31 +0100
Subject: [PATCH 16/26] setting frame resolution
---
dlclivegui/cameras/gentl_backend.py | 81 ++++++++++++++++++++++++++--
dlclivegui/cameras/opencv_backend.py | 37 +++++++++++--
2 files changed, 112 insertions(+), 6 deletions(-)
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 701d4fd..7e81f84 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -48,6 +48,10 @@ def __init__(self, settings):
self._cti_search_paths: Tuple[str, ...] = self._parse_cti_paths(
props.get("cti_search_paths")
)
+ # Parse resolution (width, height) with defaults
+ self._resolution: Optional[Tuple[int, int]] = self._parse_resolution(
+ props.get("resolution")
+ )
self._harvester = None
self._acquirer = None
@@ -232,6 +236,27 @@ def _parse_crop(self, crop) -> Optional[Tuple[int, int, int, int]]:
return tuple(int(v) for v in crop)
return None
+ def _parse_resolution(self, resolution) -> Optional[Tuple[int, int]]:
+ """Parse resolution setting.
+
+ Args:
+ resolution: Can be a tuple/list [width, height], or None
+
+ Returns:
+ Tuple of (width, height) or None if not specified
+ Default is (720, 540) if parsing fails but value is provided
+ """
+ if resolution is None:
+ return (720, 540) # Default resolution
+
+ if isinstance(resolution, (list, tuple)) and len(resolution) == 2:
+ try:
+ return (int(resolution[0]), int(resolution[1]))
+ except (ValueError, TypeError):
+ return (720, 540)
+
+ return (720, 540)
+
@staticmethod
def _search_cti_file(patterns: Tuple[str, ...]) -> Optional[str]:
"""Search for a CTI file using the given patterns.
@@ -318,9 +343,59 @@ def _configure_pixel_format(self, node_map) -> None:
pass
def _configure_resolution(self, node_map) -> None:
- # Don't configure width/height - use camera's native resolution
- # Width and height will be determined from actual frames
- pass
+ """Configure camera resolution (width and height)."""
+ if self._resolution is None:
+ return
+
+ width, height = self._resolution
+
+ # Try to set width
+ for width_attr in ("Width", "WidthMax"):
+ try:
+ node = getattr(node_map, width_attr)
+ if width_attr == "Width":
+ # Get constraints
+ try:
+ min_w = node.min
+ max_w = node.max
+ inc_w = getattr(node, 'inc', 1)
+ # Adjust to valid value
+ width = self._adjust_to_increment(width, min_w, max_w, inc_w)
+ node.value = int(width)
+ break
+ except Exception:
+ # Try setting without adjustment
+ try:
+ node.value = int(width)
+ break
+ except Exception:
+ continue
+ except AttributeError:
+ continue
+
+ # Try to set height
+ for height_attr in ("Height", "HeightMax"):
+ try:
+ node = getattr(node_map, height_attr)
+ if height_attr == "Height":
+ # Get constraints
+ try:
+ min_h = node.min
+ max_h = node.max
+ inc_h = getattr(node, 'inc', 1)
+ # Adjust to valid value
+ height = self._adjust_to_increment(height, min_h, max_h, inc_h)
+ node.value = int(height)
+ break
+ except Exception:
+ # Try setting without adjustment
+ try:
+ node.value = int(height)
+ break
+ except Exception:
+ continue
+ except AttributeError:
+ continue
def _configure_exposure(self, node_map) -> None:
if self._exposure is None:
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index f4ee01a..7face8f 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -17,6 +17,10 @@ class OpenCVCameraBackend(CameraBackend):
def __init__(self, settings):
super().__init__(settings)
self._capture: cv2.VideoCapture | None = None
+ # Parse resolution with defaults (720x540)
+ self._resolution: Tuple[int, int] = self._parse_resolution(
+ settings.properties.get("resolution")
+ )
def open(self) -> None:
backend_flag = self._resolve_backend(self.settings.properties.get("api"))
@@ -77,22 +81,49 @@ def device_name(self) -> str:
base_name = backend_name
return f"{base_name} camera #{self.settings.index}"
+ def _parse_resolution(self, resolution) -> Tuple[int, int]:
+ """Parse resolution setting.
+
+ Args:
+ resolution: Can be a tuple/list [width, height], or None
+
+ Returns:
+ Tuple of (width, height), defaults to (720, 540)
+ """
+ if resolution is None:
+ return (720, 540) # Default resolution
+
+ if isinstance(resolution, (list, tuple)) and len(resolution) == 2:
+ try:
+ return (int(resolution[0]), int(resolution[1]))
+ except (ValueError, TypeError):
+ return (720, 540)
+
+ return (720, 540)
+
def _configure_capture(self) -> None:
if self._capture is None:
return
- # Don't set width/height - capture at camera's native resolution
- # Only set FPS if specified
+
+ # Set resolution (width x height)
+ width, height = self._resolution
+ self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, float(width))
+ self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, float(height))
+
+ # Set FPS if specified
if self.settings.fps:
self._capture.set(cv2.CAP_PROP_FPS, float(self.settings.fps))
+
# Set any additional properties from the properties dict
for prop, value in self.settings.properties.items():
- if prop == "api":
+ if prop in ("api", "resolution"):
continue
try:
prop_id = int(prop)
except (TypeError, ValueError):
continue
self._capture.set(prop_id, float(value))
+
# Update actual FPS from camera
actual_fps = self._capture.get(cv2.CAP_PROP_FPS)
if actual_fps:
From 44de72325f5439caf6d4c7a7ee308395b4572fb8 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Tue, 4 Nov 2025 11:13:28 +0100
Subject: [PATCH 17/26] updated to look for model files
---
dlclivegui/gui.py | 12 +-
dlclivegui/processors/dlc_processor_socket.py | 213 ++++++++++++++++++
docs/features.md | 26 +--
docs/user_guide.md | 2 +-
4 files changed, 233 insertions(+), 20 deletions(-)
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index bd3bee7..efc4b35 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -56,7 +56,7 @@
logging.basicConfig(level=logging.INFO)
-PATH2MODELS = "C:\\Users\\User\\Repos\\DeepLabCut-live-GUI\\models"
+PATH2MODELS = "C:\\Users\\User\\Repos\\DeepLabCut-live-GUI\\dlc_training\\dlclive"
class MainWindow(QMainWindow):
@@ -266,7 +266,7 @@ def _build_dlc_group(self) -> QGroupBox:
self.model_type_combo = QComboBox()
self.model_type_combo.addItem("Base (TensorFlow)", "base")
self.model_type_combo.addItem("PyTorch", "pytorch")
- self.model_type_combo.setCurrentIndex(0) # Default to base
+ self.model_type_combo.setCurrentIndex(1) # Default to PyTorch
form.addRow("Model type", self.model_type_combo)
# Processor selection
@@ -729,11 +729,11 @@ def _save_config_to_path(self, path: Path) -> None:
self.statusBar().showMessage(f"Saved configuration to {path}", 5000)
def _action_browse_model(self) -> None:
- directory = QFileDialog.getExistingDirectory(
- self, "Select DLCLive model directory", PATH2MODELS
+ file_path, _ = QFileDialog.getOpenFileName(
+ self, "Select DLCLive model file", PATH2MODELS, "Model files (*.pt *.pb);;All files (*.*)"
)
- if directory:
- self.model_path_edit.setText(directory)
+ if file_path:
+ self.model_path_edit.setText(file_path)
def _action_browse_directory(self) -> None:
directory = QFileDialog.getExistingDirectory(
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index 3ab5866..9215c5e 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -574,11 +574,224 @@ def get_data(self):
save_dict["filter_kwargs"] = self.filter_kwargs
return save_dict
+
+
+class MyProcessorTorchmodels_socket(BaseProcessor_socket):
+ """
+ DLC Processor with pose calculations (center, heading, head angle) and optional filtering.
+
+ Calculates:
+ - center: Weighted average of head keypoints
+ - heading: Body orientation (degrees)
+ - head_angle: Head rotation relative to body (radians)
+
+ Broadcasts: [timestamp, center_x, center_y, heading, head_angle]
+ """
+
+ # Metadata for GUI discovery
+ PROCESSOR_NAME = "Mouse Pose with less keypoints"
+ PROCESSOR_DESCRIPTION = (
+ "Calculates mouse center, heading, and head angle with optional One-Euro filtering"
+ )
+ PROCESSOR_PARAMS = {
+ "bind": {
+ "type": "tuple",
+ "default": ("0.0.0.0", 6000),
+ "description": "Server address (host, port)",
+ },
+ "authkey": {
+ "type": "bytes",
+ "default": b"secret password",
+ "description": "Authentication key for clients",
+ },
+ "use_perf_counter": {
+ "type": "bool",
+ "default": False,
+ "description": "Use time.perf_counter() instead of time.time()",
+ },
+ "use_filter": {
+ "type": "bool",
+ "default": False,
+ "description": "Apply One-Euro filter to calculated values",
+ },
+ "filter_kwargs": {
+ "type": "dict",
+ "default": {"min_cutoff": 1.0, "beta": 0.02, "d_cutoff": 1.0},
+ "description": "One-Euro filter parameters (min_cutoff, beta, d_cutoff)",
+ },
+ "save_original": {
+ "type": "bool",
+ "default": False,
+ "description": "Save raw pose arrays for analysis",
+ },
+ }
+
+ def __init__(
+ self,
+ bind=("0.0.0.0", 6000),
+ authkey=b"secret password",
+ use_perf_counter=False,
+ use_filter=False,
+ filter_kwargs={},
+ save_original=False,
+ ):
+ """
+ DLC Processor with multi-client broadcasting support.
+
+ Args:
+ bind: (host, port) tuple for server binding
+ authkey: Authentication key for client connections
+ use_perf_counter: If True, use time.perf_counter() instead of time.time()
+ use_filter: If True, apply One-Euro filter to pose data
+ filter_kwargs: Dict with OneEuroFilter parameters (min_cutoff, beta, d_cutoff)
+ save_original: If True, save raw pose arrays
+ """
+ super().__init__(
+ bind=bind,
+ authkey=authkey,
+ use_perf_counter=use_perf_counter,
+ save_original=save_original,
+ )
+
+ # Additional data storage for processed values
+ self.center_x = deque()
+ self.center_y = deque()
+ self.heading_direction = deque()
+ self.head_angle = deque()
+
+ # Filtering
+ self.use_filter = use_filter
+ self.filter_kwargs = filter_kwargs
+ self.filters = None # Will be initialized on first pose
+
+ def _clear_data_queues(self):
+ """Clear all data storage queues including pose-specific ones."""
+ super()._clear_data_queues()
+ self.center_x.clear()
+ self.center_y.clear()
+ self.heading_direction.clear()
+ self.head_angle.clear()
+
+ def _initialize_filters(self, vals):
+ """Initialize One-Euro filters for each output variable."""
+ t0 = self.timing_func()
+ self.filters = {
+ "center_x": OneEuroFilter(t0, vals[0], **self.filter_kwargs),
+ "center_y": OneEuroFilter(t0, vals[1], **self.filter_kwargs),
+ "heading": OneEuroFilter(t0, vals[2], **self.filter_kwargs),
+ "head_angle": OneEuroFilter(t0, vals[3], **self.filter_kwargs),
+ }
+ LOG.debug(f"Initialized One-Euro filters with parameters: {self.filter_kwargs}")
+
+ def process(self, pose, **kwargs):
+ """
+ Process pose: calculate center/heading/head_angle, optionally filter, and broadcast.
+
+ Args:
+ pose: DLC pose array (N_keypoints x 3) with [x, y, confidence]
+ **kwargs: Additional metadata (frame_time, pose_time, etc.)
+
+ Returns:
+ pose: Unmodified pose array
+ """
+ # Save original pose if requested (from base class)
+ if self.save_original:
+ self.original_pose.append(pose.copy())
+
+ # Extract keypoints and confidence
+ xy = pose[:, :2]
+ conf = pose[:, 2]
+
+ # Calculate weighted center from head keypoints
+ head_xy = xy[[0, 1, 2, 3, 5, 6, 7], :]
+ head_conf = conf[[0, 1, 2, 3, 5, 6, 7]]
+ center = np.average(head_xy, axis=0, weights=head_conf)
+
+ neck = np.average(xy[[2, 3, 6, 7], :], axis=0, weights=conf[[2, 3, 6, 7]])
+
+ # Calculate body axis (tail_base -> neck)
+ body_axis = neck - xy[9]
+ body_axis /= sqrt(np.sum(body_axis**2))
+
+ # Calculate head axis (neck -> nose)
+ head_axis = xy[0] - neck
+ head_axis /= sqrt(np.sum(head_axis**2))
+
+ # Calculate head angle relative to body
+ cross = body_axis[0] * head_axis[1] - head_axis[0] * body_axis[1]
+ sign = copysign(1, cross) # Positive when looking left
+ try:
+ head_angle = acos(body_axis @ head_axis) * sign
+ except ValueError:
+ head_angle = 0
+
+ # Calculate heading (body orientation)
+ heading = atan2(body_axis[1], body_axis[0])
+ heading = degrees(heading)
+
+ # Raw values (heading unwrapped for filtering)
+ vals = [center[0], center[1], heading, head_angle]
+
+ # Apply filtering if enabled
+ curr_time = self.timing_func()
+ if self.use_filter:
+ if self.filters is None:
+ self._initialize_filters(vals)
+
+ # Filter each value (heading is filtered in unwrapped space)
+ filtered_vals = [
+ self.filters["center_x"](curr_time, vals[0]),
+ self.filters["center_y"](curr_time, vals[1]),
+ self.filters["heading"](curr_time, vals[2]),
+ self.filters["head_angle"](curr_time, vals[3]),
+ ]
+ vals = filtered_vals
+
+ # Wrap heading to [0, 360) after filtering
+ vals[2] = vals[2] % 360
+
+ # Update step counter
+ self.curr_step = self.curr_step + 1
+
+ # Store processed data (only if recording)
+ if self.recording:
+ self.center_x.append(vals[0])
+ self.center_y.append(vals[1])
+ self.heading_direction.append(vals[2])
+ self.head_angle.append(vals[3])
+ self.time_stamp.append(curr_time)
+ self.step.append(self.curr_step)
+ self.frame_time.append(kwargs.get("frame_time", -1))
+ if "pose_time" in kwargs:
+ self.pose_time.append(kwargs["pose_time"])
+
+ # Broadcast processed values to all connected clients
+ payload = [curr_time, vals[0], vals[1], vals[2], vals[3]]
+ self.broadcast(payload)
+
+ return pose
+
+ def get_data(self):
+ """Get logged data including base class data and processed values."""
+ # Get base class data
+ save_dict = super().get_data()
+
+ # Add processed values
+ save_dict["x_pos"] = np.array(self.center_x)
+ save_dict["y_pos"] = np.array(self.center_y)
+ save_dict["heading_direction"] = np.array(self.heading_direction)
+ save_dict["head_angle"] = np.array(self.head_angle)
+ save_dict["use_filter"] = self.use_filter
+ save_dict["filter_kwargs"] = self.filter_kwargs
+
+ return save_dict
+
# Register processors for GUI discovery
PROCESSOR_REGISTRY["BaseProcessor_socket"] = BaseProcessor_socket
PROCESSOR_REGISTRY["MyProcessor_socket"] = MyProcessor_socket
+PROCESSOR_REGISTRY["MyProcessorTorchmodels_socket"] = MyProcessorTorchmodels_socket
def get_available_processors():
diff --git a/docs/features.md b/docs/features.md
index 5fd535d..1a04068 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -84,7 +84,7 @@ The GUI intelligently detects available cameras:
Example detection output:
```
-[CameraDetection] Available cameras for backend 'gentl':
+[CameraDetection] Available cameras for backend 'gentl':
['0:DMK 37BUX287 (26320523)', '1:Basler acA1920 (40123456)']
```
@@ -127,7 +127,7 @@ Example detection output:
- **Purpose**: Visual ROI definition
- **Configuration**: (x0, y0, x1, y1) coordinates
- **Color**: Red rectangle overlay
-- **Use Cases**:
+- **Use Cases**:
- Crop region preview
- Analysis area marking
- Multi-region tracking
@@ -310,21 +310,21 @@ Custom pose processors for real-time analysis and control.
```python
class MyProcessor:
"""Custom processor example."""
-
+
def process(self, pose, timestamp):
"""Process pose data in real-time.
-
+
Args:
pose: numpy array (n_keypoints, 3) - x, y, likelihood
timestamp: float - frame timestamp
"""
# Extract keypoint positions
nose_x, nose_y = pose[0, :2]
-
+
# Custom logic
if nose_x > 320:
self.trigger_event()
-
+
# Return results (optional)
return {"position": (nose_x, nose_y)}
```
@@ -353,15 +353,15 @@ class RecordingProcessor:
def __init__(self):
self._vid_recording = False
self.session_name = "default"
-
+
@property
def video_recording(self):
return self._vid_recording
-
+
def start_recording(self, session):
self.session_name = session
self._vid_recording = True
-
+
def stop_recording(self):
self._vid_recording = False
```
@@ -423,7 +423,7 @@ The GUI monitors `video_recording` property and automatically starts/stops recor
#### Button States
- **Disabled**: Gray, not clickable
- **Enabled**: Default color, clickable
-- **Active**:
+- **Active**:
- Preview running: Stop button enabled
- Inference initializing: Blue "Initializing DLCLive!"
- Inference ready: Green "DLCLive running!"
@@ -447,7 +447,7 @@ The GUI monitors `video_recording` property and automatically starts/stops recor
#### DLC Metrics
- **Inference FPS**: Poses processed per second
-- **Latency**:
+- **Latency**:
- Last frame latency (ms)
- Average latency over session (ms)
- **Queue**: Number of frames waiting
@@ -535,7 +535,7 @@ class MyBackend(CameraBackend):
def open(self): ...
def read(self) -> Tuple[np.ndarray, float]: ...
def close(self): ...
-
+
@classmethod
def get_device_count(cls) -> int: ...
```
@@ -554,7 +554,7 @@ class MyProcessor:
def __init__(self, **kwargs):
# Initialize
pass
-
+
def process(self, pose, timestamp):
# Process pose
pass
diff --git a/docs/user_guide.md b/docs/user_guide.md
index 289bfb8..50374c0 100644
--- a/docs/user_guide.md
+++ b/docs/user_guide.md
@@ -239,7 +239,7 @@ Check **"Display pose predictions"** to overlay keypoints on video.
3. **Select format**:
- **Container**: mp4 (recommended), avi, mov
- - **Codec**:
+ - **Codec**:
- `h264_nvenc` (NVIDIA GPU - fastest)
- `libx264` (CPU - universal)
- `hevc_nvenc` (NVIDIA H.265)
From a5e70a654d1444829a2d0bbe79aa44e1a4d80353 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Wed, 5 Nov 2025 14:42:09 +0100
Subject: [PATCH 18/26] dropped tensorflow support via GUI
---
dlclivegui/config.py | 2 +-
dlclivegui/gui.py | 21 ++-------------------
2 files changed, 3 insertions(+), 20 deletions(-)
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index 126eb13..f78cae8 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -50,7 +50,7 @@ class DLCProcessorSettings:
model_path: str = ""
additional_options: Dict[str, Any] = field(default_factory=dict)
- model_type: Optional[str] = "base"
+ model_type: str = "pytorch" # Only PyTorch models are supported
@dataclass
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index efc4b35..eba2d6c 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -261,13 +261,7 @@ def _build_dlc_group(self) -> QGroupBox:
self.browse_model_button = QPushButton("Browse…")
self.browse_model_button.clicked.connect(self._action_browse_model)
path_layout.addWidget(self.browse_model_button)
- form.addRow("Model directory", path_layout)
-
- self.model_type_combo = QComboBox()
- self.model_type_combo.addItem("Base (TensorFlow)", "base")
- self.model_type_combo.addItem("PyTorch", "pytorch")
- self.model_type_combo.setCurrentIndex(1) # Default to PyTorch
- form.addRow("Model type", self.model_type_combo)
+ form.addRow("Model file", path_layout)
# Processor selection
processor_path_layout = QHBoxLayout()
@@ -481,12 +475,6 @@ def _apply_config(self, config: ApplicationSettings) -> None:
dlc = config.dlc
self.model_path_edit.setText(dlc.model_path)
- # Set model type
- model_type = dlc.model_type or "base"
- model_type_index = self.model_type_combo.findData(model_type)
- if model_type_index >= 0:
- self.model_type_combo.setCurrentIndex(model_type_index)
-
self.additional_options_edit.setPlainText(json.dumps(dlc.additional_options, indent=2))
recording = config.recording
@@ -655,13 +643,9 @@ def _parse_json(self, value: str) -> dict:
return json.loads(text)
def _dlc_settings_from_ui(self) -> DLCProcessorSettings:
- model_type = self.model_type_combo.currentData()
- if not isinstance(model_type, str):
- model_type = "base"
-
return DLCProcessorSettings(
model_path=self.model_path_edit.text().strip(),
- model_type=model_type,
+ model_type="pytorch",
additional_options=self._parse_json(self.additional_options_edit.toPlainText()),
)
@@ -925,7 +909,6 @@ def _update_dlc_controls_enabled(self) -> None:
widgets = [
self.model_path_edit,
self.browse_model_button,
- self.model_type_combo,
self.processor_folder_edit,
self.browse_processor_folder_button,
self.refresh_processors_button,
From 229a8396d27b235a10f477725a880515d35b3d3b Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Thu, 6 Nov 2025 11:51:26 +0100
Subject: [PATCH 19/26] more profiling of the processes
---
dlclivegui/dlc_processor.py | 133 +++++++++++++++++++++++++++++++++---
dlclivegui/gui.py | 110 ++++++++++++++++++++++++-----
2 files changed, 219 insertions(+), 24 deletions(-)
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 80944d8..5ce2061 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -17,6 +17,9 @@
LOGGER = logging.getLogger(__name__)
+# Enable profiling
+ENABLE_PROFILING = True
+
try: # pragma: no cover - optional dependency
from dlclive import DLCLive # type: ignore
except Exception: # pragma: no cover - handled gracefully
@@ -40,6 +43,14 @@ class ProcessorStats:
processing_fps: float = 0.0
average_latency: float = 0.0
last_latency: float = 0.0
+ # Profiling metrics
+ avg_queue_wait: float = 0.0
+ avg_inference_time: float = 0.0
+ avg_signal_emit_time: float = 0.0
+ avg_total_process_time: float = 0.0
+ # Separated timing for GPU vs socket processor
+ avg_gpu_inference_time: float = 0.0 # Pure model inference
+ avg_processor_overhead: float = 0.0 # Socket processor overhead
_SENTINEL = object()
@@ -69,6 +80,14 @@ def __init__(self) -> None:
self._latencies: deque[float] = deque(maxlen=60)
self._processing_times: deque[float] = deque(maxlen=60)
self._stats_lock = threading.Lock()
+
+ # Profiling metrics
+ self._queue_wait_times: deque[float] = deque(maxlen=60)
+ self._inference_times: deque[float] = deque(maxlen=60)
+ self._signal_emit_times: deque[float] = deque(maxlen=60)
+ self._total_process_times: deque[float] = deque(maxlen=60)
+ self._gpu_inference_times: deque[float] = deque(maxlen=60)
+ self._processor_overhead_times: deque[float] = deque(maxlen=60)
def configure(self, settings: DLCProcessorSettings, processor: Optional[Any] = None) -> None:
self._settings = settings
@@ -85,6 +104,12 @@ def reset(self) -> None:
self._frames_dropped = 0
self._latencies.clear()
self._processing_times.clear()
+ self._queue_wait_times.clear()
+ self._inference_times.clear()
+ self._signal_emit_times.clear()
+ self._total_process_times.clear()
+ self._gpu_inference_times.clear()
+ self._processor_overhead_times.clear()
def shutdown(self) -> None:
self._stop_worker()
@@ -128,6 +153,14 @@ def get_stats(self) -> ProcessorStats:
)
else:
processing_fps = 0.0
+
+ # Profiling metrics
+ avg_queue_wait = sum(self._queue_wait_times) / len(self._queue_wait_times) if self._queue_wait_times else 0.0
+ avg_inference = sum(self._inference_times) / len(self._inference_times) if self._inference_times else 0.0
+ avg_signal_emit = sum(self._signal_emit_times) / len(self._signal_emit_times) if self._signal_emit_times else 0.0
+ avg_total = sum(self._total_process_times) / len(self._total_process_times) if self._total_process_times else 0.0
+ avg_gpu = sum(self._gpu_inference_times) / len(self._gpu_inference_times) if self._gpu_inference_times else 0.0
+ avg_proc_overhead = sum(self._processor_overhead_times) / len(self._processor_overhead_times) if self._processor_overhead_times else 0.0
return ProcessorStats(
frames_enqueued=self._frames_enqueued,
@@ -137,13 +170,19 @@ def get_stats(self) -> ProcessorStats:
processing_fps=processing_fps,
average_latency=avg_latency,
last_latency=last_latency,
+ avg_queue_wait=avg_queue_wait,
+ avg_inference_time=avg_inference,
+ avg_signal_emit_time=avg_signal_emit,
+ avg_total_process_time=avg_total,
+ avg_gpu_inference_time=avg_gpu,
+ avg_processor_overhead=avg_proc_overhead,
)
def _start_worker(self, init_frame: np.ndarray, init_timestamp: float) -> None:
if self._worker_thread is not None and self._worker_thread.is_alive():
return
- self._queue = queue.Queue(maxsize=2)
+ self._queue = queue.Queue(maxsize=1)
self._stop_event.clear()
self._worker_thread = threading.Thread(
target=self._worker_loop,
@@ -179,31 +218,48 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
if not self._settings.model_path:
raise RuntimeError("No DLCLive model path configured.")
+ init_start = time.perf_counter()
options = {
"model_path": self._settings.model_path,
"model_type": self._settings.model_type,
"processor": self._processor,
"dynamic": [False, 0.5, 10],
"resize": 1.0,
+ "precision": "FP32",
}
# todo expose more parameters from settings
self._dlc = DLCLive(**options)
+
+ init_inference_start = time.perf_counter()
self._dlc.init_inference(init_frame)
+ init_inference_time = time.perf_counter() - init_inference_start
+
self._initialized = True
self.initialized.emit(True)
- LOGGER.info("DLCLive model initialized successfully")
+
+ total_init_time = time.perf_counter() - init_start
+ LOGGER.info(f"DLCLive model initialized successfully (total: {total_init_time:.3f}s, init_inference: {init_inference_time:.3f}s)")
# Process the initialization frame
enqueue_time = time.perf_counter()
+
+ inference_start = time.perf_counter()
pose = self._dlc.get_pose(init_frame, frame_time=init_timestamp)
+ inference_time = time.perf_counter() - inference_start
+
+ signal_start = time.perf_counter()
+ self.pose_ready.emit(PoseResult(pose=pose, timestamp=init_timestamp))
+ signal_time = time.perf_counter() - signal_start
+
process_time = time.perf_counter()
with self._stats_lock:
self._frames_enqueued += 1
self._frames_processed += 1
self._processing_times.append(process_time)
-
- self.pose_ready.emit(PoseResult(pose=pose, timestamp=init_timestamp))
+ if ENABLE_PROFILING:
+ self._inference_times.append(inference_time)
+ self._signal_emit_times.append(signal_time)
except Exception as exc:
LOGGER.exception("Failed to initialize DLCLive", exc_info=exc)
@@ -212,29 +268,90 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
return
# Main processing loop
+ frame_count = 0
while not self._stop_event.is_set():
+ loop_start = time.perf_counter()
+
+ # Time spent waiting for queue
+ queue_wait_start = time.perf_counter()
try:
item = self._queue.get(timeout=0.1)
except queue.Empty:
continue
+ queue_wait_time = time.perf_counter() - queue_wait_start
if item is _SENTINEL:
break
frame, timestamp, enqueue_time = item
+
try:
- start_process = time.perf_counter()
+ # Time the inference - we need to separate GPU from processor overhead
+ # If processor exists, wrap its process method to time it separately
+ processor_overhead_time = 0.0
+ gpu_inference_time = 0.0
+
+ if self._processor is not None:
+ # Wrap processor.process() to time it
+ original_process = self._processor.process
+ processor_time_holder = [0.0] # Use list to allow modification in nested scope
+
+ def timed_process(pose, **kwargs):
+ proc_start = time.perf_counter()
+ result = original_process(pose, **kwargs)
+ processor_time_holder[0] = time.perf_counter() - proc_start
+ return result
+
+ self._processor.process = timed_process
+
+ inference_start = time.perf_counter()
pose = self._dlc.get_pose(frame, frame_time=timestamp)
+ inference_time = time.perf_counter() - inference_start
+
+ if self._processor is not None:
+ # Restore original process method
+ self._processor.process = original_process
+ processor_overhead_time = processor_time_holder[0]
+ gpu_inference_time = inference_time - processor_overhead_time
+ else:
+ # No processor, all time is GPU inference
+ gpu_inference_time = inference_time
+
+ # Time the signal emission
+ signal_start = time.perf_counter()
+ self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
+ signal_time = time.perf_counter() - signal_start
+
end_process = time.perf_counter()
-
+ total_process_time = end_process - loop_start
latency = end_process - enqueue_time
with self._stats_lock:
self._frames_processed += 1
self._latencies.append(latency)
self._processing_times.append(end_process)
-
- self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
+
+ if ENABLE_PROFILING:
+ self._queue_wait_times.append(queue_wait_time)
+ self._inference_times.append(inference_time)
+ self._signal_emit_times.append(signal_time)
+ self._total_process_times.append(total_process_time)
+ self._gpu_inference_times.append(gpu_inference_time)
+ self._processor_overhead_times.append(processor_overhead_time)
+
+ # Log profiling every 100 frames
+ frame_count += 1
+ if ENABLE_PROFILING and frame_count % 100 == 0:
+ LOGGER.info(
+ f"[Profile] Frame {frame_count}: "
+ f"queue_wait={queue_wait_time*1000:.2f}ms, "
+ f"inference={inference_time*1000:.2f}ms "
+ f"(GPU={gpu_inference_time*1000:.2f}ms, processor={processor_overhead_time*1000:.2f}ms), "
+ f"signal_emit={signal_time*1000:.2f}ms, "
+ f"total={total_process_time*1000:.2f}ms, "
+ f"latency={latency*1000:.2f}ms"
+ )
+
except Exception as exc:
LOGGER.exception("Pose inference failed", exc_info=exc)
self.error.emit(str(exc))
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index eba2d6c..bb5dd1e 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -65,8 +65,26 @@ class MainWindow(QMainWindow):
def __init__(self, config: Optional[ApplicationSettings] = None):
super().__init__()
self.setWindowTitle("DeepLabCut Live GUI")
- self._config = config or DEFAULT_CONFIG
- self._config_path: Optional[Path] = None
+
+ # Try to load myconfig.json from the application directory if no config provided
+ if config is None:
+ myconfig_path = Path(__file__).parent.parent / "myconfig.json"
+ if myconfig_path.exists():
+ try:
+ config = ApplicationSettings.load(str(myconfig_path))
+ self._config_path = myconfig_path
+ logging.info(f"Loaded configuration from {myconfig_path}")
+ except Exception as exc:
+ logging.warning(f"Failed to load myconfig.json: {exc}. Using default config.")
+ config = DEFAULT_CONFIG
+ self._config_path = None
+ else:
+ config = DEFAULT_CONFIG
+ self._config_path = None
+ else:
+ self._config_path = None
+
+ self._config = config
self._current_frame: Optional[np.ndarray] = None
self._raw_frame: Optional[np.ndarray] = None
self._last_pose: Optional[PoseResult] = None
@@ -105,17 +123,67 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._metrics_timer.timeout.connect(self._update_metrics)
self._metrics_timer.start()
self._update_metrics()
+
+ # Show status message if myconfig.json was loaded
+ if self._config_path and self._config_path.name == "myconfig.json":
+ self.statusBar().showMessage(f"Auto-loaded configuration from {self._config_path}", 5000)
# ------------------------------------------------------------------ UI
def _setup_ui(self) -> None:
central = QWidget()
layout = QHBoxLayout(central)
+ # Video panel with display and performance stats
+ video_panel = QWidget()
+ video_layout = QVBoxLayout(video_panel)
+ video_layout.setContentsMargins(0, 0, 0, 0)
+
# Video display widget
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
self.video_label.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
+ video_layout.addWidget(self.video_label)
+
+ # Stats panel below video with clear labels
+ stats_widget = QWidget()
+ stats_widget.setStyleSheet("padding: 5px;")
+ stats_widget.setMinimumWidth(800) # Prevent excessive line breaks
+ stats_layout = QVBoxLayout(stats_widget)
+ stats_layout.setContentsMargins(5, 5, 5, 5)
+ stats_layout.setSpacing(3)
+
+ # Camera throughput stats
+ camera_stats_container = QHBoxLayout()
+ camera_stats_label_title = QLabel("Camera:")
+ camera_stats_container.addWidget(camera_stats_label_title)
+ self.camera_stats_label = QLabel("Camera idle")
+ self.camera_stats_label.setWordWrap(True)
+ camera_stats_container.addWidget(self.camera_stats_label)
+ camera_stats_container.addStretch(1)
+ stats_layout.addLayout(camera_stats_container)
+
+ # DLC processor stats
+ dlc_stats_container = QHBoxLayout()
+ dlc_stats_label_title = QLabel("DLC Processor:")
+ dlc_stats_container.addWidget(dlc_stats_label_title)
+ self.dlc_stats_label = QLabel("DLC processor idle")
+ self.dlc_stats_label.setWordWrap(True)
+ dlc_stats_container.addWidget(self.dlc_stats_label)
+ dlc_stats_container.addStretch(1)
+ stats_layout.addLayout(dlc_stats_container)
+
+ # Video recorder stats
+ recorder_stats_container = QHBoxLayout()
+ recorder_stats_label_title = QLabel("Recorder:")
+ recorder_stats_container.addWidget(recorder_stats_label_title)
+ self.recording_stats_label = QLabel("Recorder idle")
+ self.recording_stats_label.setWordWrap(True)
+ recorder_stats_container.addWidget(self.recording_stats_label)
+ recorder_stats_container.addStretch(1)
+ stats_layout.addLayout(recorder_stats_container)
+
+ video_layout.addWidget(stats_widget)
# Controls panel with fixed width to prevent shifting
controls_widget = QWidget()
@@ -142,9 +210,9 @@ def _setup_ui(self) -> None:
controls_layout.addWidget(button_bar_widget)
controls_layout.addStretch(1)
- # Add controls and video to main layout
+ # Add controls and video panel to main layout
layout.addWidget(controls_widget, stretch=0)
- layout.addWidget(self.video_label, stretch=1)
+ layout.addWidget(video_panel, stretch=1)
self.setCentralWidget(central)
self.setStatusBar(QStatusBar())
@@ -245,9 +313,6 @@ def _build_camera_group(self) -> QGroupBox:
self.rotation_combo.addItem("270°", 270)
form.addRow("Rotation", self.rotation_combo)
- self.camera_stats_label = QLabel("Camera idle")
- form.addRow("Throughput", self.camera_stats_label)
-
return group
def _build_dlc_group(self) -> QGroupBox:
@@ -316,10 +381,6 @@ def _build_dlc_group(self) -> QGroupBox:
self.processor_status_label.setWordWrap(True)
form.addRow("Processor Status", self.processor_status_label)
- self.dlc_stats_label = QLabel("DLC processor idle")
- self.dlc_stats_label.setWordWrap(True)
- form.addRow("Performance", self.dlc_stats_label)
-
return group
def _build_recording_group(self) -> QGroupBox:
@@ -365,10 +426,6 @@ def _build_recording_group(self) -> QGroupBox:
buttons.addWidget(self.stop_record_button)
form.addRow(recording_button_widget)
- self.recording_stats_label = QLabel(self._last_recorder_summary)
- self.recording_stats_label.setWordWrap(True)
- form.addRow("Performance", self.recording_stats_label)
-
return group
def _build_bbox_group(self) -> QGroupBox:
@@ -989,10 +1046,31 @@ def _format_dlc_stats(self, stats: ProcessorStats) -> str:
enqueue = stats.frames_enqueued
processed = stats.frames_processed
dropped = stats.frames_dropped
+
+ # Add profiling info if available
+ profile_info = ""
+ if stats.avg_inference_time > 0:
+ inf_ms = stats.avg_inference_time * 1000.0
+ queue_ms = stats.avg_queue_wait * 1000.0
+ signal_ms = stats.avg_signal_emit_time * 1000.0
+ total_ms = stats.avg_total_process_time * 1000.0
+
+ # Add GPU vs processor breakdown if available
+ gpu_breakdown = ""
+ if stats.avg_gpu_inference_time > 0 or stats.avg_processor_overhead > 0:
+ gpu_ms = stats.avg_gpu_inference_time * 1000.0
+ proc_ms = stats.avg_processor_overhead * 1000.0
+ gpu_breakdown = f" (GPU:{gpu_ms:.1f}ms+proc:{proc_ms:.1f}ms)"
+
+ profile_info = (
+ f"\n[Profile] inf:{inf_ms:.1f}ms{gpu_breakdown} queue:{queue_ms:.1f}ms "
+ f"signal:{signal_ms:.1f}ms total:{total_ms:.1f}ms"
+ )
+
return (
f"{processed}/{enqueue} frames | inference {processing_fps:.1f} fps | "
f"latency {latency_ms:.1f} ms (avg {avg_ms:.1f} ms) | "
- f"queue {stats.queue_size} | dropped {dropped}"
+ f"queue {stats.queue_size} | dropped {dropped}{profile_info}"
)
def _update_metrics(self) -> None:
From 448bdc5cd187edb74ee9bc59f9e481a43dbc54f1 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Tue, 18 Nov 2025 13:45:35 +0100
Subject: [PATCH 20/26] Add visualization settings and update camera backend
for improved pose display
---
dlclivegui/cameras/gentl_backend.py | 2 +-
dlclivegui/config.py | 20 ++-
dlclivegui/gui.py | 116 +++++++++++++++---
dlclivegui/processors/dlc_processor_socket.py | 16 ++-
pyproject.toml | 112 +++++++++++++++++
5 files changed, 248 insertions(+), 18 deletions(-)
create mode 100644 pyproject.toml
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 7e81f84..89b7dd5 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -13,7 +13,7 @@
from .base import CameraBackend
try: # pragma: no cover - optional dependency
- from harvesters.core import Harvester
+ from harvesters.core import Harvester # type: ignore
try:
from harvesters.core import HarvesterTimeoutError # type: ignore
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index f78cae8..c3a49c3 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -64,6 +64,21 @@ class BoundingBoxSettings:
y1: int = 100
+@dataclass
+class VisualizationSettings:
+ """Configuration for pose visualization."""
+
+ p_cutoff: float = 0.6 # Confidence threshold for displaying keypoints
+ colormap: str = "hot" # Matplotlib colormap for keypoints
+ bbox_color: tuple[int, int, int] = (0, 0, 255) # BGR color for bounding box (default: red)
+
+ def get_bbox_color_bgr(self) -> tuple[int, int, int]:
+ """Get bounding box color in BGR format."""
+ if isinstance(self.bbox_color, (list, tuple)) and len(self.bbox_color) == 3:
+ return tuple(int(c) for c in self.bbox_color)
+ return (0, 0, 255) # Default to red
+
+
@dataclass
class RecordingSettings:
"""Configuration for video recording."""
@@ -108,6 +123,7 @@ class ApplicationSettings:
dlc: DLCProcessorSettings = field(default_factory=DLCProcessorSettings)
recording: RecordingSettings = field(default_factory=RecordingSettings)
bbox: BoundingBoxSettings = field(default_factory=BoundingBoxSettings)
+ visualization: VisualizationSettings = field(default_factory=VisualizationSettings)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
@@ -123,7 +139,8 @@ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
recording_data.pop("options", None)
recording = RecordingSettings(**recording_data)
bbox = BoundingBoxSettings(**data.get("bbox", {}))
- return cls(camera=camera, dlc=dlc, recording=recording, bbox=bbox)
+ visualization = VisualizationSettings(**data.get("visualization", {}))
+ return cls(camera=camera, dlc=dlc, recording=recording, bbox=bbox, visualization=visualization)
def to_dict(self) -> Dict[str, Any]:
"""Serialise the configuration to a dictionary."""
@@ -133,6 +150,7 @@ def to_dict(self) -> Dict[str, Any]:
"dlc": asdict(self.dlc),
"recording": asdict(self.recording),
"bbox": asdict(self.bbox),
+ "visualization": asdict(self.visualization),
}
@classmethod
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index bb5dd1e..d1b9cf0 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -12,6 +12,7 @@
from typing import Optional
import cv2
+import matplotlib.pyplot as plt
import numpy as np
from PyQt6.QtCore import Qt, QTimer
from PyQt6.QtGui import QAction, QCloseEvent, QImage, QPixmap
@@ -47,12 +48,13 @@
CameraSettings,
DLCProcessorSettings,
RecordingSettings,
+ VisualizationSettings,
)
from dlclivegui.dlc_processor import DLCLiveProcessor, PoseResult, ProcessorStats
from dlclivegui.processors.processor_utils import instantiate_from_scan, scan_processor_folder
from dlclivegui.video_recorder import RecorderStats, VideoRecorder
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+os.environ["CUDA_VISIBLE_DEVICES"] = "1"
logging.basicConfig(level=logging.INFO)
@@ -108,6 +110,11 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._bbox_x1 = 0
self._bbox_y1 = 0
self._bbox_enabled = False
+
+ # Visualization settings (will be updated from config)
+ self._p_cutoff = 0.6
+ self._colormap = "hot"
+ self._bbox_color = (0, 0, 255) # BGR: red
self.camera_controller = CameraController()
self.dlc_processor = DLCLiveProcessor()
@@ -553,6 +560,12 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.bbox_y0_spin.setValue(bbox.y0)
self.bbox_x1_spin.setValue(bbox.x1)
self.bbox_y1_spin.setValue(bbox.y1)
+
+ # Set visualization settings from config
+ viz = config.visualization
+ self._p_cutoff = viz.p_cutoff
+ self._colormap = viz.colormap
+ self._bbox_color = viz.get_bbox_color_bgr()
def _current_config(self) -> ApplicationSettings:
return ApplicationSettings(
@@ -560,6 +573,7 @@ def _current_config(self) -> ApplicationSettings:
dlc=self._dlc_settings_from_ui(),
recording=self._recording_settings_from_ui(),
bbox=self._bbox_settings_from_ui(),
+ visualization=self._visualization_settings_from_ui(),
)
def _camera_settings_from_ui(self) -> CameraSettings:
@@ -725,6 +739,13 @@ def _bbox_settings_from_ui(self) -> BoundingBoxSettings:
y1=self.bbox_y1_spin.value(),
)
+ def _visualization_settings_from_ui(self) -> VisualizationSettings:
+ return VisualizationSettings(
+ p_cutoff=self._p_cutoff,
+ colormap=self._colormap,
+ bbox_color=self._bbox_color,
+ )
+
# ------------------------------------------------------------------ actions
def _action_load_config(self) -> None:
file_name, _ = QFileDialog.getOpenFileName(
@@ -1215,31 +1236,60 @@ def _stop_inference(self, show_message: bool = True) -> None:
# ------------------------------------------------------------------ recording
def _start_recording(self) -> None:
+ # If recorder already running, nothing to do
if self._video_recorder and self._video_recorder.is_running:
return
+
+ # If camera not running, start it automatically so frames will arrive.
+ # This allows starting recording without the user manually pressing "Start Preview".
if not self.camera_controller.is_running():
- self._show_error("Start the camera preview before recording.")
- return
- if self._current_frame is None:
- self._show_error("Wait for the first preview frame before recording.")
- return
+ try:
+ settings = self._camera_settings_from_ui()
+ except ValueError as exc:
+ self._show_error(str(exc))
+ return
+ # Store active settings and start camera preview in background
+ self._active_camera_settings = settings
+ self.camera_controller.start(settings)
+ self.preview_button.setEnabled(False)
+ self.stop_preview_button.setEnabled(True)
+ self._current_frame = None
+ self._raw_frame = None
+ self._last_pose = None
+ self._dlc_active = False
+ self._camera_frame_times.clear()
+ self._last_display_time = 0.0
+ if hasattr(self, "camera_stats_label"):
+ self.camera_stats_label.setText("Camera starting…")
+ self.statusBar().showMessage("Starting camera preview…", 3000)
+ self._update_inference_buttons()
+ self._update_camera_controls_enabled()
recording = self._recording_settings_from_ui()
if not recording.enabled:
self._show_error("Recording is disabled in the configuration.")
return
+
+ # If we already have a current frame, use its shape to set the recorder stream.
+ # Otherwise start the recorder without a fixed frame_size and configure it
+ # once the first frame arrives (see _on_frame_ready).
frame = self._current_frame
- assert frame is not None
- height, width = frame.shape[:2]
+ if frame is not None:
+ height, width = frame.shape[:2]
+ frame_size = (height, width)
+ else:
+ frame_size = None
+
frame_rate = (
self._active_camera_settings.fps
if self._active_camera_settings is not None
else self.camera_fps.value()
)
+
output_path = recording.output_path()
self._video_recorder = VideoRecorder(
output_path,
- frame_size=(height, width), # Use numpy convention: (height, width)
- frame_rate=float(frame_rate),
+ frame_size=frame_size, # None allowed; will be configured on first frame
+ frame_rate=float(frame_rate) if frame_rate is not None else None,
codec=recording.codec,
crf=recording.crf,
)
@@ -1295,7 +1345,30 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
frame = np.ascontiguousarray(frame)
self._current_frame = frame
self._track_camera_frame()
+ # If recorder is running but was started without a fixed frame_size, configure
+ # the stream now that we know the actual frame dimensions.
if self._video_recorder and self._video_recorder.is_running:
+ # Configure stream if recorder was started without a frame_size
+ try:
+ current_frame_size = getattr(self._video_recorder, "_frame_size", None)
+ except Exception:
+ current_frame_size = None
+ if current_frame_size is None:
+ try:
+ fps_value = (
+ self._active_camera_settings.fps
+ if self._active_camera_settings is not None
+ else self.camera_fps.value()
+ )
+ except Exception:
+ fps_value = None
+ h, w = frame.shape[:2]
+ try:
+ # configure_stream expects (height, width)
+ self._video_recorder.configure_stream((h, w), float(fps_value) if fps_value is not None else None)
+ except Exception:
+ # Non-fatal: continue and attempt to write anyway
+ pass
try:
success = self._video_recorder.write(frame, timestamp=frame_data.timestamp)
if not success:
@@ -1421,20 +1494,35 @@ def _draw_bbox(self, frame: np.ndarray) -> np.ndarray:
x1 = max(x0 + 1, min(x1, width))
y1 = max(y0 + 1, min(y1, height))
- # Draw red rectangle (BGR format: red is (0, 0, 255))
- cv2.rectangle(overlay, (x0, y0), (x1, y1), (0, 0, 255), 2)
+ # Draw rectangle with configured color
+ cv2.rectangle(overlay, (x0, y0), (x1, y1), self._bbox_color, 2)
return overlay
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
- for keypoint in np.asarray(pose):
+
+ # Get the colormap from config
+ cmap = plt.get_cmap(self._colormap)
+ num_keypoints = len(np.asarray(pose))
+
+ for idx, keypoint in enumerate(np.asarray(pose)):
if len(keypoint) < 2:
continue
x, y = keypoint[:2]
+ confidence = keypoint[2] if len(keypoint) > 2 else 1.0
if np.isnan(x) or np.isnan(y):
continue
- cv2.circle(overlay, (int(x), int(y)), 4, (0, 255, 0), -1)
+ if confidence < self._p_cutoff:
+ continue
+
+ # Get color from colormap (cycle through 0 to 1)
+ color_normalized = idx / max(num_keypoints - 1, 1)
+ rgba = cmap(color_normalized)
+ # Convert from RGBA [0, 1] to BGR [0, 255] for OpenCV
+ bgr_color = (int(rgba[2] * 255), int(rgba[1] * 255), int(rgba[0] * 255))
+
+ cv2.circle(overlay, (int(x), int(y)), 4, bgr_color, -1)
return overlay
def _on_dlc_initialised(self, success: bool) -> None:
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index 9215c5e..8caf31f 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -6,6 +6,7 @@
from math import acos, atan2, copysign, degrees, pi, sqrt
from multiprocessing.connection import Listener
from threading import Event, Thread
+from pathlib import Path
import numpy as np
from dlclive import Processor # type: ignore
@@ -346,7 +347,9 @@ def save(self, file=None):
LOG.info(f"Saving data to {file}")
try:
save_dict = self.get_data()
- pickle.dump(save_dict, open(file, "wb"))
+ path2save = Path(__file__).parent.parent.parent / "data" / file
+ LOG.info(f"Path should be {path2save}")
+ pickle.dump(file, open(path2save, "wb"))
save_code = 1
except Exception as e:
LOG.error(f"Save failed: {e}")
@@ -634,6 +637,7 @@ def __init__(
use_filter=False,
filter_kwargs={},
save_original=False,
+ p_cutoff=0.4,
):
"""
DLC Processor with multi-client broadcasting support.
@@ -659,6 +663,8 @@ def __init__(
self.heading_direction = deque()
self.head_angle = deque()
+ self.p_cutoff = p_cutoff
+
# Filtering
self.use_filter = use_filter
self.filter_kwargs = filter_kwargs
@@ -705,7 +711,13 @@ def process(self, pose, **kwargs):
# Calculate weighted center from head keypoints
head_xy = xy[[0, 1, 2, 3, 5, 6, 7], :]
head_conf = conf[[0, 1, 2, 3, 5, 6, 7]]
- center = np.average(head_xy, axis=0, weights=head_conf)
+ # set low confidence keypoints to zero weight
+ head_conf = np.where(head_conf < self.p_cutoff, 0, head_conf)
+ try:
+ center = np.average(head_xy, axis=0, weights=head_conf)
+ except ZeroDivisionError:
+ # If all keypoints have zero weight, return without processing
+ return pose
neck = np.average(xy[[2, 3, 6, 7], :], axis=0, weights=conf[[2, 3, 6, 7]])
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..edbb9d0
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,112 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deeplabcut-live-gui"
+version = "2.0"
+description = "PyQt-based GUI to run real time DeepLabCut experiments"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "GNU Lesser General Public License v3 (LGPLv3)"}
+authors = [
+ {name = "A. & M. Mathis Labs", email = "adim@deeplabcut.org"}
+]
+keywords = ["deeplabcut", "pose estimation", "real-time", "gui", "deep learning"]
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+ "Operating System :: OS Independent",
+ "Development Status :: 4 - Beta",
+ "Intended Audience :: Science/Research",
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+dependencies = [
+ "deeplabcut-live",
+ "PyQt6",
+ "numpy",
+ "opencv-python",
+ "vidgear[core]",
+ "matplotlib",
+]
+
+[project.optional-dependencies]
+basler = ["pypylon"]
+gentl = ["harvesters"]
+all = ["pypylon", "harvesters"]
+dev = [
+ "pytest>=7.0",
+ "pytest-cov>=4.0",
+ "pytest-mock>=3.10",
+ "pytest-qt>=4.2",
+ "black>=23.0",
+ "flake8>=6.0",
+ "mypy>=1.0",
+]
+test = [
+ "pytest>=7.0",
+ "pytest-cov>=4.0",
+ "pytest-mock>=3.10",
+ "pytest-qt>=4.2",
+]
+
+[project.urls]
+Homepage = "https://github.com/DeepLabCut/DeepLabCut-live-GUI"
+Repository = "https://github.com/DeepLabCut/DeepLabCut-live-GUI"
+Documentation = "https://github.com/DeepLabCut/DeepLabCut-live-GUI"
+"Bug Tracker" = "https://github.com/DeepLabCut/DeepLabCut-live-GUI/issues"
+
+[project.scripts]
+dlclivegui = "dlclivegui.gui:main"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["dlclivegui*"]
+exclude = ["tests*", "docs*"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+ "-v",
+ "--strict-markers",
+ "--tb=short",
+ "--cov=dlclivegui",
+ "--cov-report=term-missing",
+ "--cov-report=html",
+]
+markers = [
+ "unit: Unit tests for individual components",
+ "integration: Integration tests for component interaction",
+ "functional: Functional tests for end-to-end workflows",
+ "slow: Tests that take a long time to run",
+ "gui: Tests that require GUI interaction",
+]
+
+[tool.coverage.run]
+source = ["dlclivegui"]
+omit = [
+ "*/tests/*",
+ "*/__pycache__/*",
+ "*/site-packages/*",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+ "pragma: no cover",
+ "def __repr__",
+ "raise AssertionError",
+ "raise NotImplementedError",
+ "if __name__ == .__main__.:",
+ "if TYPE_CHECKING:",
+ "@abstract",
+]
From 246452f5484f8eb3f6c63874ace1b5a77eac849a Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 5 Dec 2025 12:01:17 +0100
Subject: [PATCH 21/26] small bug fixes
---
dlclivegui/processors/GUI_INTEGRATION.md | 167 ------------------
dlclivegui/processors/dlc_processor_socket.py | 2 +-
dlclivegui/video_recorder.py | 16 +-
docs/install.md | 25 ---
4 files changed, 9 insertions(+), 201 deletions(-)
delete mode 100644 dlclivegui/processors/GUI_INTEGRATION.md
delete mode 100644 docs/install.md
diff --git a/dlclivegui/processors/GUI_INTEGRATION.md b/dlclivegui/processors/GUI_INTEGRATION.md
deleted file mode 100644
index 6e504f1..0000000
--- a/dlclivegui/processors/GUI_INTEGRATION.md
+++ /dev/null
@@ -1,167 +0,0 @@
-# GUI Integration Guide
-
-## Quick Answer
-
-Here's how to use `scan_processor_folder` in your GUI:
-
-```python
-from example_gui_usage import scan_processor_folder, instantiate_from_scan
-
-# 1. Scan folder
-all_processors = scan_processor_folder("./processors")
-
-# 2. Populate dropdown with keys (for backend) and display names (for user)
-for key, info in all_processors.items():
- # key = "file.py::ClassName" (use this for instantiation)
- # display_name = "Human Name (file.py)" (show this to user)
- display_name = f"{info['name']} ({info['file']})"
- dropdown.add_item(key, display_name)
-
-# 3. When user selects, get the key from dropdown
-selected_key = dropdown.get_selected_value() # e.g., "dlc_processor_socket.py::MyProcessor_socket"
-
-# 4. Get processor info
-processor_info = all_processors[selected_key]
-
-# 5. Build parameter form from processor_info['params']
-for param_name, param_info in processor_info['params'].items():
- add_input_field(param_name, param_info['type'], param_info['default'])
-
-# 6. When user clicks Create, instantiate using the key
-user_params = get_form_values()
-processor = instantiate_from_scan(all_processors, selected_key, **user_params)
-```
-
-## The Key Insight
-
-**The key returned by `scan_processor_folder` is what you use to instantiate!**
-
-```python
-# OLD problem: "I have a name, how do I load it?"
-# NEW solution: Use the key directly
-
-all_processors = scan_processor_folder(folder)
-# Returns: {"file.py::ClassName": {processor_info}, ...}
-
-# The KEY "file.py::ClassName" uniquely identifies the processor
-# Pass this key to instantiate_from_scan()
-
-processor = instantiate_from_scan(all_processors, "file.py::ClassName", **params)
-```
-
-## What's in the returned dict?
-
-```python
-all_processors = {
- "dlc_processor_socket.py::MyProcessor_socket": {
- "class": , # The actual class
- "name": "Mouse Pose Processor", # Human-readable name
- "description": "Calculates mouse...", # Description
- "params": { # All parameters
- "bind": {
- "type": "tuple",
- "default": ("0.0.0.0", 6000),
- "description": "Server address"
- },
- # ... more parameters
- },
- "file": "dlc_processor_socket.py", # Source file
- "class_name": "MyProcessor_socket", # Class name
- "file_path": "/full/path/to/file.py" # Full path
- }
-}
-```
-
-## GUI Workflow
-
-### Step 1: Scan Folder
-```python
-all_processors = scan_processor_folder("./processors")
-```
-
-### Step 2: Populate Dropdown
-```python
-# Store keys in order (for mapping dropdown index -> key)
-self.processor_keys = list(all_processors.keys())
-
-# Create display names for dropdown
-display_names = [
- f"{info['name']} ({info['file']})"
- for info in all_processors.values()
-]
-dropdown.set_items(display_names)
-```
-
-### Step 3: User Selects Processor
-```python
-def on_processor_selected(dropdown_index):
- # Get the key
- key = self.processor_keys[dropdown_index]
-
- # Get processor info
- info = all_processors[key]
-
- # Show description
- description_label.text = info['description']
-
- # Build parameter form
- for param_name, param_info in info['params'].items():
- add_parameter_field(
- name=param_name,
- type=param_info['type'],
- default=param_info['default'],
- help_text=param_info['description']
- )
-```
-
-### Step 4: User Clicks Create
-```python
-def on_create_clicked():
- # Get selected key
- key = self.processor_keys[dropdown.current_index]
-
- # Get user's parameter values
- user_params = parameter_form.get_values()
-
- # Instantiate using the key!
- self.processor = instantiate_from_scan(
- all_processors,
- key,
- **user_params
- )
-
- print(f"Created: {self.processor.__class__.__name__}")
-```
-
-## Why This Works
-
-1. **Unique Keys**: `"file.py::ClassName"` format ensures uniqueness even if multiple files have same class name
-
-2. **All Info Included**: Each dict entry has everything needed (class, metadata, parameters)
-
-3. **Simple Lookup**: Just use the key to get processor info or instantiate
-
-4. **No Manual Imports**: `scan_processor_folder` handles all module loading
-
-5. **Type Safety**: Parameter metadata includes types for validation
-
-## Complete Example
-
-See `processor_gui.py` for a full working tkinter GUI that demonstrates:
-- Folder scanning
-- Processor selection
-- Parameter form generation
-- Instantiation
-
-Run it with:
-```bash
-python processor_gui.py
-```
-
-## Files
-
-- `dlc_processor_socket.py` - Processors with metadata and registry
-- `example_gui_usage.py` - Scanning and instantiation functions + examples
-- `processor_gui.py` - Full tkinter GUI
-- `GUI_USAGE_GUIDE.py` - Pseudocode and examples
-- `README.md` - Documentation on the plugin system
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index 8caf31f..73a5cb4 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -349,7 +349,7 @@ def save(self, file=None):
save_dict = self.get_data()
path2save = Path(__file__).parent.parent.parent / "data" / file
LOG.info(f"Path should be {path2save}")
- pickle.dump(file, open(path2save, "wb"))
+ pickle.dump(save_dict, open(path2save, "wb"))
save_code = 1
except Exception as e:
LOG.error(f"Save failed: {e}")
diff --git a/dlclivegui/video_recorder.py b/dlclivegui/video_recorder.py
index a40ed28..47cfff4 100644
--- a/dlclivegui/video_recorder.py
+++ b/dlclivegui/video_recorder.py
@@ -27,14 +27,14 @@
class RecorderStats:
"""Snapshot of recorder throughput metrics."""
- frames_enqueued: int
- frames_written: int
- dropped_frames: int
- queue_size: int
- average_latency: float
- last_latency: float
- write_fps: float
- buffer_seconds: float
+ frames_enqueued: int = 0
+ frames_written: int = 0
+ dropped_frames: int = 0
+ queue_size: int = 0
+ average_latency: float = 0.0
+ last_latency: float = 0.0
+ write_fps: float = 0.0
+ buffer_seconds: float = 0.0
_SENTINEL = object()
diff --git a/docs/install.md b/docs/install.md
deleted file mode 100644
index 24ef4f4..0000000
--- a/docs/install.md
+++ /dev/null
@@ -1,25 +0,0 @@
-## Installation Instructions
-
-### Windows or Linux Desktop
-
-We recommend that you install DeepLabCut-live in a conda environment. First, please install Anaconda:
-- [Windows](https://docs.anaconda.com/anaconda/install/windows/)
-- [Linux](https://docs.anaconda.com/anaconda/install/linux/)
-
-Create a conda environment with python 3.7 and tensorflow:
-```
-conda create -n dlc-live python=3.7 tensorflow-gpu==1.13.1 # if using GPU
-conda create -n dlc-live python=3.7 tensorflow==1.13.1 # if not using GPU
-```
-
-Activate the conda environment and install the DeepLabCut-live package:
-```
-conda activate dlc-live
-pip install deeplabcut-live-gui
-```
-
-### NVIDIA Jetson Development Kit
-
-First, please refer to our complete instructions for [installing DeepLabCut-Live! on a NVIDIA Jetson Development Kit](https://github.com/DeepLabCut/DeepLabCut-live/blob/master/docs/install_jetson.md).
-
-Next, install the DeepLabCut-live-GUI: `pip install deeplabcut-live-gui`.
From 103399c0a98631d8d96e44948cdd9e45608b96bf Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 5 Dec 2025 13:36:34 +0100
Subject: [PATCH 22/26] black formatting
---
dlclivegui/cameras/gentl_backend.py | 20 ++---
dlclivegui/cameras/opencv_backend.py | 16 ++--
dlclivegui/config.py | 4 +-
dlclivegui/dlc_processor.py | 78 ++++++++++++-------
dlclivegui/gui.py | 51 ++++++------
dlclivegui/processors/dlc_processor_socket.py | 29 ++++---
6 files changed, 116 insertions(+), 82 deletions(-)
diff --git a/dlclivegui/cameras/gentl_backend.py b/dlclivegui/cameras/gentl_backend.py
index 89b7dd5..cdc798e 100644
--- a/dlclivegui/cameras/gentl_backend.py
+++ b/dlclivegui/cameras/gentl_backend.py
@@ -13,7 +13,7 @@
from .base import CameraBackend
try: # pragma: no cover - optional dependency
- from harvesters.core import Harvester # type: ignore
+ from harvesters.core import Harvester # type: ignore
try:
from harvesters.core import HarvesterTimeoutError # type: ignore
@@ -238,23 +238,23 @@ def _parse_crop(self, crop) -> Optional[Tuple[int, int, int, int]]:
def _parse_resolution(self, resolution) -> Optional[Tuple[int, int]]:
"""Parse resolution setting.
-
+
Args:
resolution: Can be a tuple/list [width, height], or None
-
+
Returns:
Tuple of (width, height) or None if not specified
Default is (720, 540) if parsing fails but value is provided
"""
if resolution is None:
return (720, 540) # Default resolution
-
+
if isinstance(resolution, (list, tuple)) and len(resolution) == 2:
try:
return (int(resolution[0]), int(resolution[1]))
except (ValueError, TypeError):
return (720, 540)
-
+
return (720, 540)
@staticmethod
@@ -346,9 +346,9 @@ def _configure_resolution(self, node_map) -> None:
"""Configure camera resolution (width and height)."""
if self._resolution is None:
return
-
+
width, height = self._resolution
-
+
# Try to set width
for width_attr in ("Width", "WidthMax"):
try:
@@ -358,7 +358,7 @@ def _configure_resolution(self, node_map) -> None:
try:
min_w = node.min
max_w = node.max
- inc_w = getattr(node, 'inc', 1)
+ inc_w = getattr(node, "inc", 1)
# Adjust to valid value
width = self._adjust_to_increment(width, min_w, max_w, inc_w)
node.value = int(width)
@@ -372,7 +372,7 @@ def _configure_resolution(self, node_map) -> None:
continue
except AttributeError:
continue
-
+
# Try to set height
for height_attr in ("Height", "HeightMax"):
try:
@@ -382,7 +382,7 @@ def _configure_resolution(self, node_map) -> None:
try:
min_h = node.min
max_h = node.max
- inc_h = getattr(node, 'inc', 1)
+ inc_h = getattr(node, "inc", 1)
# Adjust to valid value
height = self._adjust_to_increment(height, min_h, max_h, inc_h)
node.value = int(height)
diff --git a/dlclivegui/cameras/opencv_backend.py b/dlclivegui/cameras/opencv_backend.py
index 7face8f..651eeab 100644
--- a/dlclivegui/cameras/opencv_backend.py
+++ b/dlclivegui/cameras/opencv_backend.py
@@ -83,37 +83,37 @@ def device_name(self) -> str:
def _parse_resolution(self, resolution) -> Tuple[int, int]:
"""Parse resolution setting.
-
+
Args:
resolution: Can be a tuple/list [width, height], or None
-
+
Returns:
Tuple of (width, height), defaults to (720, 540)
"""
if resolution is None:
return (720, 540) # Default resolution
-
+
if isinstance(resolution, (list, tuple)) and len(resolution) == 2:
try:
return (int(resolution[0]), int(resolution[1]))
except (ValueError, TypeError):
return (720, 540)
-
+
return (720, 540)
def _configure_capture(self) -> None:
if self._capture is None:
return
-
+
# Set resolution (width x height)
width, height = self._resolution
self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, float(width))
self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, float(height))
-
+
# Set FPS if specified
if self.settings.fps:
self._capture.set(cv2.CAP_PROP_FPS, float(self.settings.fps))
-
+
# Set any additional properties from the properties dict
for prop, value in self.settings.properties.items():
if prop in ("api", "resolution"):
@@ -123,7 +123,7 @@ def _configure_capture(self) -> None:
except (TypeError, ValueError):
continue
self._capture.set(prop_id, float(value))
-
+
# Update actual FPS from camera
actual_fps = self._capture.get(cv2.CAP_PROP_FPS)
if actual_fps:
diff --git a/dlclivegui/config.py b/dlclivegui/config.py
index c3a49c3..7cc629a 100644
--- a/dlclivegui/config.py
+++ b/dlclivegui/config.py
@@ -140,7 +140,9 @@ def from_dict(cls, data: Dict[str, Any]) -> "ApplicationSettings":
recording = RecordingSettings(**recording_data)
bbox = BoundingBoxSettings(**data.get("bbox", {}))
visualization = VisualizationSettings(**data.get("visualization", {}))
- return cls(camera=camera, dlc=dlc, recording=recording, bbox=bbox, visualization=visualization)
+ return cls(
+ camera=camera, dlc=dlc, recording=recording, bbox=bbox, visualization=visualization
+ )
def to_dict(self) -> Dict[str, Any]:
"""Serialise the configuration to a dictionary."""
diff --git a/dlclivegui/dlc_processor.py b/dlclivegui/dlc_processor.py
index 5ce2061..03a9b70 100644
--- a/dlclivegui/dlc_processor.py
+++ b/dlclivegui/dlc_processor.py
@@ -80,7 +80,7 @@ def __init__(self) -> None:
self._latencies: deque[float] = deque(maxlen=60)
self._processing_times: deque[float] = deque(maxlen=60)
self._stats_lock = threading.Lock()
-
+
# Profiling metrics
self._queue_wait_times: deque[float] = deque(maxlen=60)
self._inference_times: deque[float] = deque(maxlen=60)
@@ -153,14 +153,38 @@ def get_stats(self) -> ProcessorStats:
)
else:
processing_fps = 0.0
-
+
# Profiling metrics
- avg_queue_wait = sum(self._queue_wait_times) / len(self._queue_wait_times) if self._queue_wait_times else 0.0
- avg_inference = sum(self._inference_times) / len(self._inference_times) if self._inference_times else 0.0
- avg_signal_emit = sum(self._signal_emit_times) / len(self._signal_emit_times) if self._signal_emit_times else 0.0
- avg_total = sum(self._total_process_times) / len(self._total_process_times) if self._total_process_times else 0.0
- avg_gpu = sum(self._gpu_inference_times) / len(self._gpu_inference_times) if self._gpu_inference_times else 0.0
- avg_proc_overhead = sum(self._processor_overhead_times) / len(self._processor_overhead_times) if self._processor_overhead_times else 0.0
+ avg_queue_wait = (
+ sum(self._queue_wait_times) / len(self._queue_wait_times)
+ if self._queue_wait_times
+ else 0.0
+ )
+ avg_inference = (
+ sum(self._inference_times) / len(self._inference_times)
+ if self._inference_times
+ else 0.0
+ )
+ avg_signal_emit = (
+ sum(self._signal_emit_times) / len(self._signal_emit_times)
+ if self._signal_emit_times
+ else 0.0
+ )
+ avg_total = (
+ sum(self._total_process_times) / len(self._total_process_times)
+ if self._total_process_times
+ else 0.0
+ )
+ avg_gpu = (
+ sum(self._gpu_inference_times) / len(self._gpu_inference_times)
+ if self._gpu_inference_times
+ else 0.0
+ )
+ avg_proc_overhead = (
+ sum(self._processor_overhead_times) / len(self._processor_overhead_times)
+ if self._processor_overhead_times
+ else 0.0
+ )
return ProcessorStats(
frames_enqueued=self._frames_enqueued,
@@ -229,28 +253,30 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
}
# todo expose more parameters from settings
self._dlc = DLCLive(**options)
-
+
init_inference_start = time.perf_counter()
self._dlc.init_inference(init_frame)
init_inference_time = time.perf_counter() - init_inference_start
-
+
self._initialized = True
self.initialized.emit(True)
-
+
total_init_time = time.perf_counter() - init_start
- LOGGER.info(f"DLCLive model initialized successfully (total: {total_init_time:.3f}s, init_inference: {init_inference_time:.3f}s)")
+ LOGGER.info(
+ f"DLCLive model initialized successfully (total: {total_init_time:.3f}s, init_inference: {init_inference_time:.3f}s)"
+ )
# Process the initialization frame
enqueue_time = time.perf_counter()
-
+
inference_start = time.perf_counter()
pose = self._dlc.get_pose(init_frame, frame_time=init_timestamp)
inference_time = time.perf_counter() - inference_start
-
+
signal_start = time.perf_counter()
self.pose_ready.emit(PoseResult(pose=pose, timestamp=init_timestamp))
signal_time = time.perf_counter() - signal_start
-
+
process_time = time.perf_counter()
with self._stats_lock:
@@ -271,7 +297,7 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
frame_count = 0
while not self._stop_event.is_set():
loop_start = time.perf_counter()
-
+
# Time spent waiting for queue
queue_wait_start = time.perf_counter()
try:
@@ -284,30 +310,30 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None:
break
frame, timestamp, enqueue_time = item
-
+
try:
# Time the inference - we need to separate GPU from processor overhead
# If processor exists, wrap its process method to time it separately
processor_overhead_time = 0.0
gpu_inference_time = 0.0
-
+
if self._processor is not None:
# Wrap processor.process() to time it
original_process = self._processor.process
processor_time_holder = [0.0] # Use list to allow modification in nested scope
-
+
def timed_process(pose, **kwargs):
proc_start = time.perf_counter()
result = original_process(pose, **kwargs)
processor_time_holder[0] = time.perf_counter() - proc_start
return result
-
+
self._processor.process = timed_process
-
+
inference_start = time.perf_counter()
pose = self._dlc.get_pose(frame, frame_time=timestamp)
inference_time = time.perf_counter() - inference_start
-
+
if self._processor is not None:
# Restore original process method
self._processor.process = original_process
@@ -321,7 +347,7 @@ def timed_process(pose, **kwargs):
signal_start = time.perf_counter()
self.pose_ready.emit(PoseResult(pose=pose, timestamp=timestamp))
signal_time = time.perf_counter() - signal_start
-
+
end_process = time.perf_counter()
total_process_time = end_process - loop_start
latency = end_process - enqueue_time
@@ -330,7 +356,7 @@ def timed_process(pose, **kwargs):
self._frames_processed += 1
self._latencies.append(latency)
self._processing_times.append(end_process)
-
+
if ENABLE_PROFILING:
self._queue_wait_times.append(queue_wait_time)
self._inference_times.append(inference_time)
@@ -338,7 +364,7 @@ def timed_process(pose, **kwargs):
self._total_process_times.append(total_process_time)
self._gpu_inference_times.append(gpu_inference_time)
self._processor_overhead_times.append(processor_overhead_time)
-
+
# Log profiling every 100 frames
frame_count += 1
if ENABLE_PROFILING and frame_count % 100 == 0:
@@ -351,7 +377,7 @@ def timed_process(pose, **kwargs):
f"total={total_process_time*1000:.2f}ms, "
f"latency={latency*1000:.2f}ms"
)
-
+
except Exception as exc:
LOGGER.exception("Pose inference failed", exc_info=exc)
self.error.emit(str(exc))
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index d1b9cf0..c4d5781 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -67,7 +67,7 @@ class MainWindow(QMainWindow):
def __init__(self, config: Optional[ApplicationSettings] = None):
super().__init__()
self.setWindowTitle("DeepLabCut Live GUI")
-
+
# Try to load myconfig.json from the application directory if no config provided
if config is None:
myconfig_path = Path(__file__).parent.parent / "myconfig.json"
@@ -85,7 +85,7 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._config_path = None
else:
self._config_path = None
-
+
self._config = config
self._current_frame: Optional[np.ndarray] = None
self._raw_frame: Optional[np.ndarray] = None
@@ -110,7 +110,7 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._bbox_x1 = 0
self._bbox_y1 = 0
self._bbox_enabled = False
-
+
# Visualization settings (will be updated from config)
self._p_cutoff = 0.6
self._colormap = "hot"
@@ -130,10 +130,12 @@ def __init__(self, config: Optional[ApplicationSettings] = None):
self._metrics_timer.timeout.connect(self._update_metrics)
self._metrics_timer.start()
self._update_metrics()
-
+
# Show status message if myconfig.json was loaded
if self._config_path and self._config_path.name == "myconfig.json":
- self.statusBar().showMessage(f"Auto-loaded configuration from {self._config_path}", 5000)
+ self.statusBar().showMessage(
+ f"Auto-loaded configuration from {self._config_path}", 5000
+ )
# ------------------------------------------------------------------ UI
def _setup_ui(self) -> None:
@@ -144,14 +146,14 @@ def _setup_ui(self) -> None:
video_panel = QWidget()
video_layout = QVBoxLayout(video_panel)
video_layout.setContentsMargins(0, 0, 0, 0)
-
+
# Video display widget
self.video_label = QLabel("Camera preview not started")
self.video_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
self.video_label.setMinimumSize(640, 360)
self.video_label.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Expanding)
video_layout.addWidget(self.video_label)
-
+
# Stats panel below video with clear labels
stats_widget = QWidget()
stats_widget.setStyleSheet("padding: 5px;")
@@ -159,7 +161,7 @@ def _setup_ui(self) -> None:
stats_layout = QVBoxLayout(stats_widget)
stats_layout.setContentsMargins(5, 5, 5, 5)
stats_layout.setSpacing(3)
-
+
# Camera throughput stats
camera_stats_container = QHBoxLayout()
camera_stats_label_title = QLabel("Camera:")
@@ -169,7 +171,7 @@ def _setup_ui(self) -> None:
camera_stats_container.addWidget(self.camera_stats_label)
camera_stats_container.addStretch(1)
stats_layout.addLayout(camera_stats_container)
-
+
# DLC processor stats
dlc_stats_container = QHBoxLayout()
dlc_stats_label_title = QLabel("DLC Processor:")
@@ -179,7 +181,7 @@ def _setup_ui(self) -> None:
dlc_stats_container.addWidget(self.dlc_stats_label)
dlc_stats_container.addStretch(1)
stats_layout.addLayout(dlc_stats_container)
-
+
# Video recorder stats
recorder_stats_container = QHBoxLayout()
recorder_stats_label_title = QLabel("Recorder:")
@@ -189,7 +191,7 @@ def _setup_ui(self) -> None:
recorder_stats_container.addWidget(self.recording_stats_label)
recorder_stats_container.addStretch(1)
stats_layout.addLayout(recorder_stats_container)
-
+
video_layout.addWidget(stats_widget)
# Controls panel with fixed width to prevent shifting
@@ -560,7 +562,7 @@ def _apply_config(self, config: ApplicationSettings) -> None:
self.bbox_y0_spin.setValue(bbox.y0)
self.bbox_x1_spin.setValue(bbox.x1)
self.bbox_y1_spin.setValue(bbox.y1)
-
+
# Set visualization settings from config
viz = config.visualization
self._p_cutoff = viz.p_cutoff
@@ -792,7 +794,10 @@ def _save_config_to_path(self, path: Path) -> None:
def _action_browse_model(self) -> None:
file_path, _ = QFileDialog.getOpenFileName(
- self, "Select DLCLive model file", PATH2MODELS, "Model files (*.pt *.pb);;All files (*.*)"
+ self,
+ "Select DLCLive model file",
+ PATH2MODELS,
+ "Model files (*.pt *.pb);;All files (*.*)",
)
if file_path:
self.model_path_edit.setText(file_path)
@@ -1067,7 +1072,7 @@ def _format_dlc_stats(self, stats: ProcessorStats) -> str:
enqueue = stats.frames_enqueued
processed = stats.frames_processed
dropped = stats.frames_dropped
-
+
# Add profiling info if available
profile_info = ""
if stats.avg_inference_time > 0:
@@ -1075,19 +1080,19 @@ def _format_dlc_stats(self, stats: ProcessorStats) -> str:
queue_ms = stats.avg_queue_wait * 1000.0
signal_ms = stats.avg_signal_emit_time * 1000.0
total_ms = stats.avg_total_process_time * 1000.0
-
+
# Add GPU vs processor breakdown if available
gpu_breakdown = ""
if stats.avg_gpu_inference_time > 0 or stats.avg_processor_overhead > 0:
gpu_ms = stats.avg_gpu_inference_time * 1000.0
proc_ms = stats.avg_processor_overhead * 1000.0
gpu_breakdown = f" (GPU:{gpu_ms:.1f}ms+proc:{proc_ms:.1f}ms)"
-
+
profile_info = (
f"\n[Profile] inf:{inf_ms:.1f}ms{gpu_breakdown} queue:{queue_ms:.1f}ms "
f"signal:{signal_ms:.1f}ms total:{total_ms:.1f}ms"
)
-
+
return (
f"{processed}/{enqueue} frames | inference {processing_fps:.1f} fps | "
f"latency {latency_ms:.1f} ms (avg {avg_ms:.1f} ms) | "
@@ -1365,7 +1370,9 @@ def _on_frame_ready(self, frame_data: FrameData) -> None:
h, w = frame.shape[:2]
try:
# configure_stream expects (height, width)
- self._video_recorder.configure_stream((h, w), float(fps_value) if fps_value is not None else None)
+ self._video_recorder.configure_stream(
+ (h, w), float(fps_value) if fps_value is not None else None
+ )
except Exception:
# Non-fatal: continue and attempt to write anyway
pass
@@ -1501,11 +1508,11 @@ def _draw_bbox(self, frame: np.ndarray) -> np.ndarray:
def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
overlay = frame.copy()
-
+
# Get the colormap from config
cmap = plt.get_cmap(self._colormap)
num_keypoints = len(np.asarray(pose))
-
+
for idx, keypoint in enumerate(np.asarray(pose)):
if len(keypoint) < 2:
continue
@@ -1515,13 +1522,13 @@ def _draw_pose(self, frame: np.ndarray, pose: np.ndarray) -> np.ndarray:
continue
if confidence < self._p_cutoff:
continue
-
+
# Get color from colormap (cycle through 0 to 1)
color_normalized = idx / max(num_keypoints - 1, 1)
rgba = cmap(color_normalized)
# Convert from RGBA [0, 1] to BGR [0, 255] for OpenCV
bgr_color = (int(rgba[2] * 255), int(rgba[1] * 255), int(rgba[0] * 255))
-
+
cv2.circle(overlay, (int(x), int(y)), 4, bgr_color, -1)
return overlay
diff --git a/dlclivegui/processors/dlc_processor_socket.py b/dlclivegui/processors/dlc_processor_socket.py
index 73a5cb4..1ec9827 100644
--- a/dlclivegui/processors/dlc_processor_socket.py
+++ b/dlclivegui/processors/dlc_processor_socket.py
@@ -5,8 +5,8 @@
from collections import deque
from math import acos, atan2, copysign, degrees, pi, sqrt
from multiprocessing.connection import Listener
-from threading import Event, Thread
from pathlib import Path
+from threading import Event, Thread
import numpy as np
from dlclive import Processor # type: ignore
@@ -229,10 +229,10 @@ def _handle_client_message(self, msg):
elif cmd == "set_filter":
# Handle filter enable/disable (subclasses override if they support filtering)
use_filter = msg.get("use_filter", False)
- if hasattr(self, 'use_filter'):
+ if hasattr(self, "use_filter"):
self.use_filter = bool(use_filter)
# Reset filters to reinitialize with new setting
- if hasattr(self, 'filters'):
+ if hasattr(self, "filters"):
self.filters = None
LOG.info(f"Filtering {'enabled' if use_filter else 'disabled'}")
else:
@@ -241,11 +241,11 @@ def _handle_client_message(self, msg):
elif cmd == "set_filter_params":
# Handle filter parameter updates (subclasses override if they support filtering)
filter_kwargs = msg.get("filter_kwargs", {})
- if hasattr(self, 'filter_kwargs'):
+ if hasattr(self, "filter_kwargs"):
# Update filter parameters
self.filter_kwargs.update(filter_kwargs)
# Reset filters to reinitialize with new parameters
- if hasattr(self, 'filters'):
+ if hasattr(self, "filters"):
self.filters = None
LOG.info(f"Filter parameters updated: {filter_kwargs}")
else:
@@ -315,10 +315,10 @@ def process(self, pose, **kwargs):
def stop(self):
"""Stop the processor and close all connections."""
LOG.info("Stopping processor...")
-
+
# Signal stop to all threads
self._stop.set()
-
+
# Close all client connections first
for c in list(self.conns):
try:
@@ -326,18 +326,18 @@ def stop(self):
except Exception:
pass
self.conns.discard(c)
-
+
# Close the listener socket
- if hasattr(self, 'listener') and self.listener:
+ if hasattr(self, "listener") and self.listener:
try:
self.listener.close()
except Exception as e:
LOG.debug(f"Error closing listener: {e}")
-
+
# Give the OS time to release the socket on Windows
# This prevents WinError 10048 when restarting
time.sleep(0.1)
-
+
LOG.info("Processor stopped, all connections closed")
def save(self, file=None):
@@ -349,7 +349,7 @@ def save(self, file=None):
save_dict = self.get_data()
path2save = Path(__file__).parent.parent.parent / "data" / file
LOG.info(f"Path should be {path2save}")
- pickle.dump(save_dict, open(path2save, "wb"))
+ pickle.dump(save_dict, open(path2save, "wb"))
save_code = 1
except Exception as e:
LOG.error(f"Save failed: {e}")
@@ -577,7 +577,7 @@ def get_data(self):
save_dict["filter_kwargs"] = self.filter_kwargs
return save_dict
-
+
class MyProcessorTorchmodels_socket(BaseProcessor_socket):
"""
@@ -716,7 +716,7 @@ def process(self, pose, **kwargs):
try:
center = np.average(head_xy, axis=0, weights=head_conf)
except ZeroDivisionError:
- # If all keypoints have zero weight, return without processing
+ # If all keypoints have zero weight, return without processing
return pose
neck = np.average(xy[[2, 3, 6, 7], :], axis=0, weights=conf[[2, 3, 6, 7]])
@@ -797,7 +797,6 @@ def get_data(self):
save_dict["filter_kwargs"] = self.filter_kwargs
return save_dict
-
# Register processors for GUI discovery
From bcd89cc0ec12c4029296dc94fbf71a1d692d59cd Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 5 Dec 2025 16:15:49 +0100
Subject: [PATCH 23/26] some docs
---
docs/camera_support.md | 135 +---------------
docs/features.md | 184 +--------------------
docs/user_guide.md | 353 +----------------------------------------
3 files changed, 5 insertions(+), 667 deletions(-)
diff --git a/docs/camera_support.md b/docs/camera_support.md
index 4d9ba22..ed0b1e0 100644
--- a/docs/camera_support.md
+++ b/docs/camera_support.md
@@ -42,10 +42,6 @@ You can select the backend in the GUI from the "Backend" dropdown, or in your co
- **OpenCV compatible cameras**: For webcams and compatible USB cameras.
- **Aravis backend**: For GenICam/GigE Vision cameras (requires Homebrew installation).
-#### NVIDIA Jetson
-- **OpenCV compatible cameras**: Standard V4L2 camera support.
-- **Aravis backend**: Supported but may have platform-specific bugs. See [Aravis issues](https://github.com/AravisProject/aravis/issues/324).
-
### Quick Installation Guide
#### Aravis (Linux/Ubuntu)
@@ -67,10 +63,8 @@ Install vendor-provided camera drivers and SDK. CTI files are typically in:
| Feature | OpenCV | GenTL | Aravis | Basler (pypylon) |
|---------|--------|-------|--------|------------------|
-| Ease of use | ⭐⭐⭐⭐⭐ | ⭐⭐⭐ | ⭐⭐⭐⭐ | ⭐⭐⭐ |
-| Auto-detection | Basic | Yes | Yes | Yes |
-| Exposure control | Limited | Yes | Yes | Yes |
-| Gain control | Limited | Yes | Yes | Yes |
+| Exposure control | No | Yes | Yes | Yes |
+| Gain control | No | Yes | Yes | Yes |
| Windows | ✅ | ✅ | ❌ | ✅ |
| Linux | ✅ | ✅ | ✅ | ✅ |
| macOS | ✅ | ❌ | ✅ | ✅ |
@@ -80,128 +74,3 @@ Install vendor-provided camera drivers and SDK. CTI files are typically in:
- [Aravis Backend](aravis_backend.md) - GenICam/GigE cameras on Linux/macOS
- GenTL Backend - Industrial cameras via vendor CTI files
- OpenCV Backend - Universal webcam support
-
-### Contributing New Camera Types
-
-Any camera that can be accessed through python (e.g. if the company offers a python package) can be integrated into the DeepLabCut-live-GUI. To contribute, please build off of our [base `Camera` class](../dlclivegui/camera/camera.py), and please use our [currently supported cameras](../dlclivegui/camera) as examples.
-
-New camera classes must inherit our base camera class, and provide at least two arguments:
-
-- id: an arbitrary name for a camera
-- resolution: the image size
-
-Other common options include:
-
-- exposure
-- gain
-- rotate
-- crop
-- fps
-
-If the camera does not have it's own display module, you can use our Tkinter video display built into the DeepLabCut-live-GUI by passing `use_tk_display=True` to the base camera class, and control the size of the displayed image using the `display_resize` parameter (`display_resize=1` for full image, `display_resize=0.5` to display images at half the width and height of recorded images).
-
-Here is an example of a camera that allows users to set the resolution, exposure, and crop, and uses the Tkinter display:
-
-```python
-from dlclivegui import Camera
-
-class MyNewCamera(Camera)
-
- def __init__(self, id="", resolution=[640, 480], exposure=0, crop=None, display_resize=1):
- super().__init__(id,
- resolution=resolution,
- exposure=exposure,
- crop=crop,
- use_tk_display=True,
- display_resize=display_resize)
-
-```
-
-All arguments of your camera's `__init__` method will be available to edit in the GUI's `Edit Camera Settings` window. To ensure that you pass arguments of the correct data type, it is helpful to provide default values for each argument of the correct data type (e.g. if `myarg` is a string, please use `myarg=""` instead of `myarg=None`). If a certain argument has only a few possible values, and you want to limit the options user's can input into the `Edit Camera Settings` window, please implement a `@static_method` called `arg_restrictions`. This method should return a dictionary where the keys are the arguments for which you want to provide value restrictions, and the values are the possible values that a specific argument can take on. Below is an example that restrictions the values for `use_tk_display` to `True` or `False`, and restricts the possible values of `resolution` to `[640, 480]` or `[320, 240]`.
-
-```python
- @static_method
- def arg_restrictions():
- return {'use_tk_display' : [True, False],
- 'resolution' : [[640, 480], [320, 240]]}
-```
-
-In addition to an `__init__` method that calls the `dlclivegui.Camera.__init__` method, you need to overwrite the `dlclivegui.Camera.set_capture_device`, `dlclive.Camera.close_capture_device`, and one of the following two methods: `dlclivegui.Camera.get_image` or `dlclivegui.Camera.get_image_on_time`.
-
-Your camera class's `set_capture_device` method should open the camera feed and confirm that the appropriate settings (such as exposure, rotation, gain, etc.) have been properly set. The `close_capture_device` method should simply close the camera stream. For example, see the [OpenCV camera](../dlclivegui/camera/opencv.py) `set_capture_device` and `close_capture_device` method.
-
-If you're camera has built in methods to ensure the correct frame rate (e.g. when grabbing images, it will block until the next image is ready), then overwrite the `get_image_on_time` method. If the camera does not block until the next image is ready, then please set the `get_image` method, and the base camera class's `get_image_on_time` method will ensure that images are only grabbed at the specified frame rate.
-
-The `get_image` method has no input arguments, but must return an image as a numpy array. We also recommend converting images to 8-bit integers (data type `uint8`).
-
-The `get_image_on_time` method has no input arguments, but must return an image as a numpy array (as in `get_image`) and the timestamp at which the image is returned (using python's `time.time()` function).
-
-### Camera Specific Tips for Installation & Use:
-
-#### Basler cameras
-
-Basler USB3 cameras are compatible with Aravis. However, integration with DeepLabCut-live-GUI can also be obtained with `pypylon`, the python module to drive Basler cameras, and supported by the company. Please note using `pypylon` requires you to install Pylon viewer, a free of cost GUI also developed and supported by Basler and available on several platforms.
-
-* **Pylon viewer**: https://www.baslerweb.com/en/sales-support/downloads/software-downloads/#type=pylonsoftware;language=all;version=all
-* `pypylon`: https://github.com/basler/pypylon/releases
-
-If you want to use DeepLabCut-live-GUI with a Basler USB3 camera via pypylon, see the folllowing instructions. Please note this is tested on Ubuntu 20.04. It may (or may not) work similarly in other platforms (contributed by [@antortjim](https://github.com/antortjim)). This procedure should take around 10 minutes:
-
-**Install Pylon viewer**
-
-1. Download .deb file
-Download the .deb file in the downloads center of Basler. Last version as of writing this was **pylon 6.2.0 Camera Software Suite Linux x86 (64 Bit) - Debian Installer Package**.
-
-
-2. Install .deb file
-
-```
-sudo dpkg -i pylon_6.2.0.21487-deb0_amd64.deb
-```
-
-**Install swig**
-
-Required for compilation of non python code within pypylon
-
-1. Install swig dependencies
-
-You may have to install these in a fresh Ubuntu 20.04 install
-
-```
-sudo apt install gcc g++
-sudo apt install libpcre3-dev
-sudo apt install make
-```
-
-2. Download swig
-
-Go to http://prdownloads.sourceforge.net/swig/swig-4.0.2.tar.gz and download the tar gz
-
-3. Install swig
-```
-tar -zxvf swig-4.0.2.tar.gz
-cd swig-4.0.2
-./configure
-make
-sudo make install
-```
-
-**Install pypylon**
-
-1. Download pypylon
-
-```
-wget https://github.com/basler/pypylon/archive/refs/tags/1.7.2.tar.gz
-```
-
-or go to https://github.com/basler/pypylon/releases and get the version you want!
-
-2. Install pypylon
-
-```
-tar -zxvf 1.7.2.tar.gz
-cd pypylon-1.7.2
-python setup.py install
-```
-
-Once you have completed these steps, you should be able to call your Basler camera from DeepLabCut-live-GUI using the BaslerCam camera type that appears after clicking "Add camera")
diff --git a/docs/features.md b/docs/features.md
index 1a04068..91d87ad 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -20,7 +20,7 @@
The GUI supports four different camera backends, each optimized for different use cases:
#### OpenCV Backend
-- **Platform**: Windows, Linux, macOS
+- **Platform**: Windows, Linux
- **Best For**: Webcams, simple USB cameras
- **Installation**: Built-in with OpenCV
- **Limitations**: Limited exposure/gain control
@@ -32,10 +32,9 @@ The GUI supports four different camera backends, each optimized for different us
- **Features**: Full camera control, smart device detection
#### Aravis Backend
-- **Platform**: Linux (best), macOS
+- **Platform**: Linux (best)
- **Best For**: GenICam/GigE Vision cameras
- **Installation**: System packages (`gir1.2-aravis-0.8`)
-- **Features**: Excellent Linux support, native GigE
#### Basler Backend (pypylon)
- **Platform**: Windows, Linux, macOS
@@ -95,7 +94,6 @@ Example detection output:
### DLCLive Integration
#### Model Support
-- **TensorFlow (Base)**: Original DeepLabCut models
- **PyTorch**: PyTorch-exported models
- Model selection via dropdown
- Automatic model validation
@@ -240,63 +238,6 @@ Single JSON file contains all settings:
- Default values for missing fields
- Error messages for invalid entries
- Safe fallback to defaults
-
-### Configuration Sections
-
-#### Camera Settings (`camera`)
-```json
-{
- "name": "Camera 0",
- "index": 0,
- "fps": 60.0,
- "backend": "gentl",
- "exposure": 10000,
- "gain": 5.0,
- "crop_x0": 0,
- "crop_y0": 0,
- "crop_x1": 0,
- "crop_y1": 0,
- "max_devices": 3,
- "properties": {}
-}
-```
-
-#### DLC Settings (`dlc`)
-```json
-{
- "model_path": "/path/to/model",
- "model_type": "base",
- "additional_options": {
- "resize": 0.5,
- "processor": "cpu",
- "pcutoff": 0.6
- }
-}
-```
-
-#### Recording Settings (`recording`)
-```json
-{
- "enabled": true,
- "directory": "~/Videos/dlc",
- "filename": "session.mp4",
- "container": "mp4",
- "codec": "h264_nvenc",
- "crf": 23
-}
-```
-
-#### Bounding Box Settings (`bbox`)
-```json
-{
- "enabled": false,
- "x0": 0,
- "y0": 0,
- "x1": 200,
- "y1": 100
-}
-```
-
---
## Processor System
@@ -462,21 +403,6 @@ The GUI monitors `video_recording` property and automatically starts/stops recor
- **Dropped**: Encoding failures
- **Format**: "1500/1502 frames | write 59.8 fps | latency 12.3 ms (avg 12.5 ms) | queue 5 (~83 ms) | dropped 2"
-### Performance Optimization
-
-#### Automatic Adjustments
-- Frame display throttling (25 Hz max)
-- Queue backpressure handling
-- Automatic resolution detection
-
-#### User Adjustments
-- Reduce camera FPS
-- Enable ROI cropping
-- Use hardware encoding
-- Increase CRF value
-- Disable pose visualization
-- Adjust buffer counts
-
---
## Advanced Features
@@ -528,38 +454,6 @@ Qt signals/slots ensure thread-safe communication.
### Extensibility
-#### Custom Backends
-Implement `CameraBackend` abstract class:
-```python
-class MyBackend(CameraBackend):
- def open(self): ...
- def read(self) -> Tuple[np.ndarray, float]: ...
- def close(self): ...
-
- @classmethod
- def get_device_count(cls) -> int: ...
-```
-
-Register in `factory.py`:
-```python
-_BACKENDS = {
- "mybackend": ("module.path", "MyBackend")
-}
-```
-
-#### Custom Processors
-Place in `processors/` directory:
-```python
-class MyProcessor:
- def __init__(self, **kwargs):
- # Initialize
- pass
-
- def process(self, pose, timestamp):
- # Process pose
- pass
-```
-
### Debugging Features
#### Logging
@@ -567,58 +461,7 @@ class MyProcessor:
- Frame acquisition logging
- Performance warnings
- Connection status
-
-#### Development Mode
-- Syntax validation: `python -m compileall dlclivegui`
-- Type checking: `mypy dlclivegui`
-- Test files included
-
---
-
-## Use Case Examples
-
-### High-Speed Behavior Tracking
-
-**Setup**:
-- Camera: GenTL industrial camera @ 120 FPS
-- Codec: h264_nvenc (GPU encoding)
-- Crop: Region of interest only
-- DLC: PyTorch model on GPU
-
-**Settings**:
-```json
-{
- "camera": {"fps": 120, "crop_x0": 200, "crop_y0": 100, "crop_x1": 800, "crop_y1": 600},
- "recording": {"codec": "h264_nvenc", "crf": 28},
- "dlc": {"additional_options": {"processor": "gpu", "resize": 0.5}}
-}
-```
-
-### Event-Triggered Recording
-
-**Setup**:
-- Processor: Socket processor with auto-record
-- Trigger: Remote computer sends START/STOP commands
-- Session naming: Unique per trial
-
-**Workflow**:
-1. Enable "Auto-record video on processor command"
-2. Start preview and inference
-3. Remote system connects via socket
-4. Sends `START_RECORDING:trial_001` → recording starts
-5. Sends `STOP_RECORDING` → recording stops
-6. Files saved as `trial_001.mp4`
-
-### Multi-Camera Synchronization
-
-**Setup**:
-- Multiple GUI instances
-- Shared trigger signal
-- Synchronized filenames
-
-**Configuration**:
-Each instance with different camera index but same settings template.
-
---
## Keyboard Shortcuts
@@ -627,27 +470,4 @@ Each instance with different camera index but same settings template.
- **Ctrl+S**: Save configuration
- **Ctrl+Shift+S**: Save configuration as
- **Ctrl+Q**: Quit application
-
---
-
-## Platform-Specific Notes
-
-### Windows
-- Best GenTL support (vendor CTI files)
-- NVENC highly recommended
-- DirectShow backend for webcams
-
-### Linux
-- Best Aravis support (native GigE)
-- V4L2 backend for webcams
-- NVENC available with proprietary drivers
-
-### macOS
-- Limited industrial camera support
-- Aravis via Homebrew
-- Software encoding recommended
-
-### NVIDIA Jetson
-- Optimized for edge deployment
-- Hardware encoding available
-- Some Aravis compatibility issues
diff --git a/docs/user_guide.md b/docs/user_guide.md
index 50374c0..0346d53 100644
--- a/docs/user_guide.md
+++ b/docs/user_guide.md
@@ -8,10 +8,6 @@ Complete walkthrough for using the DeepLabCut-live-GUI application.
2. [Camera Setup](#camera-setup)
3. [DLCLive Configuration](#dlclive-configuration)
4. [Recording Videos](#recording-videos)
-5. [Working with Configurations](#working-with-configurations)
-6. [Common Workflows](#common-workflows)
-7. [Tips and Best Practices](#tips-and-best-practices)
-
---
## Getting Started
@@ -164,22 +160,9 @@ Select if camera is mounted at an angle:
- Model weights (`.pb`, `.pth`, etc.)
### Step 2: Choose Model Type
-
-Select from dropdown:
-- **Base (TensorFlow)**: Standard DLC models
+We only support newer, pytorch based models.
- **PyTorch**: PyTorch-based models (requires PyTorch)
-### Step 3: Configure Options (Optional)
-
-Click in "Additional options" field and enter JSON:
-
-```json
-{
- "processor": "gpu",
- "resize": 0.5,
- "pcutoff": 0.6
-}
-```
**Common options**:
- `processor`: "cpu" or "gpu"
@@ -278,13 +261,6 @@ Check **"Display pose predictions"** to overlay keypoints on video.
- Full resolution
- Sufficient disk space
-#### Long Duration Recording
-
-**Tips**:
-- Use CRF 23-25 to balance quality/size
-- Monitor disk space
-- Consider splitting into multiple files
-- Use fast SSD storage
### Auto-Recording
@@ -295,325 +271,7 @@ Enable automatic recording triggered by processor events:
3. **Start inference**: Processor will control recording
4. **Session management**: Files named by processor
-**Use cases**:
-- Trial-based experiments
-- Event-triggered recording
-- Remote control via socket processor
-- Conditional data capture
-
----
-
-## Working with Configurations
-
-### Saving Current Settings
-
-**Save** (overwrites existing file):
-1. File → Save configuration (or Ctrl+S)
-2. If no file loaded, prompts for location
-
-**Save As** (create new file):
-1. File → Save configuration as… (or Ctrl+Shift+S)
-2. Choose location and filename
-3. Enter name (e.g., `mouse_experiment.json`)
-4. Click Save
-
-### Loading Saved Settings
-
-1. File → Load configuration… (or Ctrl+O)
-2. Navigate to configuration file
-3. Select `.json` file
-4. Click Open
-5. All GUI fields update automatically
-
-### Managing Multiple Configurations
-
-**Recommended structure**:
-```
-configs/
-├── default.json # Base settings
-├── mouse_arena1.json # Arena-specific
-├── mouse_arena2.json
-├── rat_setup.json
-└── high_speed.json # Performance-specific
-```
-
-**Workflow**:
-1. Create base configuration with common settings
-2. Save variants for different:
- - Animals/subjects
- - Experimental setups
- - Camera positions
- - Recording quality levels
-
-### Configuration Templates
-
-#### Webcam + CPU Processing
-```json
-{
- "camera": {
- "backend": "opencv",
- "index": 0,
- "fps": 30.0
- },
- "dlc": {
- "model_type": "base",
- "additional_options": {"processor": "cpu"}
- },
- "recording": {
- "codec": "libx264",
- "crf": 23
- }
-}
-```
-
-#### Industrial Camera + GPU
-```json
-{
- "camera": {
- "backend": "gentl",
- "index": 0,
- "fps": 60.0,
- "exposure": 10000,
- "gain": 8.0
- },
- "dlc": {
- "model_type": "pytorch",
- "additional_options": {
- "processor": "gpu",
- "resize": 0.5
- }
- },
- "recording": {
- "codec": "h264_nvenc",
- "crf": 23
- }
-}
-```
-
----
-
-## Common Workflows
-
-### Workflow 1: Simple Webcam Tracking
-
-**Goal**: Track mouse behavior with webcam
-
-1. **Camera Setup**:
- - Backend: opencv
- - Camera: Built-in webcam (index 0)
- - FPS: 30
-
-2. **Start Preview**: Verify mouse is visible
-
-3. **Load DLC Model**: Browse to mouse tracking model
-
-4. **Start Inference**: Enable pose estimation
-
-5. **Verify Tracking**: Enable pose visualization
-
-6. **Record Trial**: Start/stop recording as needed
-
-### Workflow 2: High-Speed Industrial Camera
-
-**Goal**: Track fast movements at 120 FPS
-
-1. **Camera Setup**:
- - Backend: gentl or aravis
- - Refresh and select camera
- - FPS: 120
- - Exposure: 4000 μs (short exposure)
- - Crop: Region of interest only
-
-2. **Start Preview**: Check FPS is stable
-
-3. **Configure Recording**:
- - Codec: h264_nvenc
- - CRF: 28
- - Output: Fast SSD
-
-4. **Load DLC Model** (if needed):
- - PyTorch model
- - GPU processor
- - Resize: 0.5 (reduce load)
-
-5. **Start Recording**: Begin data capture
-
-6. **Monitor Performance**: Watch for dropped frames
-
-### Workflow 3: Event-Triggered Recording
-
-**Goal**: Record only during specific events
-
-1. **Camera Setup**: Configure as normal
-
-2. **Processor Setup**:
- - Select socket processor
- - Enable "Auto-record video on processor command"
-
-3. **Start Preview**: Camera running
-
-4. **Start Inference**: DLC + processor active
-
-5. **Remote Control**:
- - Connect to socket (default port 5000)
- - Send `START_RECORDING:trial_001`
- - Recording starts automatically
- - Send `STOP_RECORDING`
- - Recording stops, file saved
-
-### Workflow 4: Multi-Subject Tracking
-
-**Goal**: Track multiple animals simultaneously
-
-**Option A: Single Camera, Multiple Keypoints**
-1. Use DLC model trained for multiple subjects
-2. Single GUI instance
-3. Processor distinguishes subjects
-
-**Option B: Multiple Cameras**
-1. Launch multiple GUI instances
-2. Each with different camera index
-3. Synchronized configurations
-4. Coordinated filenames
-
----
-
-## Tips and Best Practices
-
-### Camera Tips
-
-1. **Lighting**:
- - Consistent, diffuse lighting
- - Avoid shadows and reflections
- - IR lighting for night vision
-
-2. **Positioning**:
- - Stable mount (minimize vibration)
- - Appropriate angle for markers
- - Sufficient field of view
-
-3. **Settings**:
- - Start with auto exposure/gain
- - Adjust manually if needed
- - Test different FPS rates
- - Use cropping to reduce load
-
-### Recording Tips
-
-1. **File Management**:
- - Use descriptive filenames
- - Include date/subject/trial info
- - Organize by experiment/session
- - Regular backups
-
-2. **Performance**:
- - Close unnecessary applications
- - Monitor disk space
- - Use SSD for high-speed recording
- - Enable GPU encoding if available
-
-3. **Quality**:
- - Test CRF values beforehand
- - Balance quality vs. file size
- - Consider post-processing needs
- - Verify recordings occasionally
-
-### DLCLive Tips
-
-1. **Model Selection**:
- - Use model trained on similar conditions
- - Test offline before live use
- - Consider resize for speed
- - GPU highly recommended
-
-2. **Performance**:
- - Monitor inference FPS
- - Check latency values
- - Watch queue depth
- - Reduce resolution if needed
-
-3. **Validation**:
- - Enable visualization initially
- - Verify tracking quality
- - Check all keypoints
- - Test edge cases
-
-### General Best Practices
-
-1. **Configuration Management**:
- - Save configurations frequently
- - Version control config files
- - Document custom settings
- - Share team configurations
-
-2. **Testing**:
- - Test setup before experiments
- - Run trial recordings
- - Verify all components
- - Check file outputs
-
-3. **Troubleshooting**:
- - Check status messages
- - Monitor performance metrics
- - Review error dialogs carefully
- - Restart if issues persist
-
-4. **Data Organization**:
- - Consistent naming scheme
- - Separate folders per session
- - Include metadata files
- - Regular data validation
-
---
-
-## Troubleshooting Guide
-
-### Camera Issues
-
-**Problem**: Camera not detected
-- **Solution**: Click Refresh, check connections, verify drivers
-
-**Problem**: Low frame rate
-- **Solution**: Reduce resolution, increase exposure, check CPU usage
-
-**Problem**: Image too dark/bright
-- **Solution**: Adjust exposure and gain settings
-
-### DLCLive Issues
-
-**Problem**: Model fails to load
-- **Solution**: Verify path, check model type, install dependencies
-
-**Problem**: Slow inference
-- **Solution**: Enable GPU, reduce resolution, use resize option
-
-**Problem**: Poor tracking
-- **Solution**: Check lighting, enable visualization, verify model quality
-
-### Recording Issues
-
-**Problem**: Dropped frames
-- **Solution**: Use GPU encoding, increase CRF, reduce FPS
-
-**Problem**: Large file sizes
-- **Solution**: Increase CRF value, use better codec
-
-**Problem**: Recording won't start
-- **Solution**: Check disk space, verify path permissions
-
----
-
-## Keyboard Reference
-
-| Action | Shortcut |
-|--------|----------|
-| Load configuration | Ctrl+O |
-| Save configuration | Ctrl+S |
-| Save configuration as | Ctrl+Shift+S |
-| Quit application | Ctrl+Q |
-
----
-
## Next Steps
- Explore [Features Documentation](features.md) for detailed capabilities
@@ -622,12 +280,3 @@ configs/
- See [Aravis Backend](aravis_backend.md) for Linux industrial cameras
---
-
-## Getting Help
-
-If you encounter issues:
-1. Check status messages in GUI
-2. Review this user guide
-3. Consult technical documentation
-4. Check GitHub issues
-5. Contact support team
From 7b70c24dac9bdc77ee304de4719917fc50d83330 Mon Sep 17 00:00:00 2001
From: Artur Schneider
Date: Fri, 5 Dec 2025 16:19:04 +0100
Subject: [PATCH 24/26] updated docs
---
README.md | 67 +++++--------------------------------------------------
1 file changed, 5 insertions(+), 62 deletions(-)
diff --git a/README.md b/README.md
index 34a2682..a330e0b 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ A modern PyQt6 GUI for running [DeepLabCut-live](https://github.com/DeepLabCut/D
- **Live Preview**: Real-time camera feed with rotation support (0°, 90°, 180°, 270°)
### DLCLive Features
-- **Model Support**: TensorFlow (base) and PyTorch models
+- **Model Support**: Only PyTorch models! (in theory also tensorflow models work)
- **Processor System**: Plugin architecture for custom pose processing
- **Auto-Recording**: Automatic video recording triggered by processor commands
- **Performance Metrics**: Real-time FPS, latency, and queue monitoring
@@ -141,11 +141,7 @@ The GUI uses a single JSON configuration file containing all experiment settings
},
"dlc": {
"model_path": "/path/to/exported-model",
- "model_type": "base",
- "additional_options": {
- "resize": 0.5,
- "processor": "cpu"
- }
+ "model_type": "pytorch",
},
"recording": {
"enabled": true,
@@ -206,34 +202,11 @@ All GUI fields are automatically synchronized with the configuration file.
"index": 0,
"fps": 60.0,
"exposure": 15000,
- "gain": 8.0,
- "properties": {
- "cti_file": "C:\\Path\\To\\Producer.cti",
- "serial_number": "12345678",
- "pixel_format": "Mono8"
- }
+ "gain": 8.0,
}
}
```
-#### Aravis
-```json
-{
- "camera": {
- "backend": "aravis",
- "index": 0,
- "fps": 60.0,
- "exposure": 10000,
- "gain": 5.0,
- "properties": {
- "camera_id": "TheImagingSource-12345678",
- "pixel_format": "Mono8",
- "n_buffers": 10,
- "timeout": 2000000
- }
- }
-}
-```
See [Camera Backend Documentation](docs/camera_support.md) for detailed setup instructions.
@@ -241,10 +214,9 @@ See [Camera Backend Documentation](docs/camera_support.md) for detailed setup in
### Model Types
-The GUI supports both TensorFlow and PyTorch DLCLive models:
+The GUI supports PyTorch DLCLive models:
-1. **Base (TensorFlow)**: Original DLC models exported for live inference
-2. **PyTorch**: PyTorch-based models (requires PyTorch installation)
+1. **PyTorch**: PyTorch-based models (requires PyTorch installation)
Select the model type from the dropdown before starting inference.
@@ -284,15 +256,6 @@ Enable "Auto-record video on processor command" to automatically start/stop reco
4. **Disable Visualization**: Uncheck "Display pose predictions" during recording
5. **Crop Region**: Use cropping to reduce frame size before inference
-### Recommended Settings by FPS
-
-| FPS Range | Codec | CRF | Buffers | Notes |
-|-----------|-------|-----|---------|-------|
-| 30-60 | libx264 | 23 | 10 | Standard quality |
-| 60-120 | h264_nvenc | 23 | 15 | GPU encoding |
-| 120-200 | h264_nvenc | 28 | 20 | Higher compression |
-| 200+ | h264_nvenc | 30 | 30 | Max performance |
-
### Project Structure
```
@@ -315,26 +278,6 @@ dlclivegui/
└── dlc_processor_socket.py
```
-### Running Tests
-
-```bash
-# Syntax check
-python -m compileall dlclivegui
-
-# Type checking (optional)
-mypy dlclivegui
-
-```
-
-### Adding New Camera Backends
-
-1. Create new backend inheriting from `CameraBackend`
-2. Implement required methods: `open()`, `read()`, `close()`
-3. Optional: Implement `get_device_count()` for smart detection
-4. Register in `cameras/factory.py`
-
-See [Camera Backend Development](docs/camera_support.md) for detailed instructions.
-
## Documentation
From 94ccc7b581f5e91fb344429cef1e1c5988cd079a Mon Sep 17 00:00:00 2001
From: Mackenzie Mathis
Date: Fri, 5 Dec 2025 17:48:34 +0100
Subject: [PATCH 25/26] Update dlclivegui/gui.py
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
---
dlclivegui/gui.py | 23 -----------------------
1 file changed, 23 deletions(-)
diff --git a/dlclivegui/gui.py b/dlclivegui/gui.py
index c4d5781..9c27a1b 100644
--- a/dlclivegui/gui.py
+++ b/dlclivegui/gui.py
@@ -686,29 +686,6 @@ def _current_camera_index_value(self) -> Optional[int]:
self.camera_index.setEditText("")
self.camera_index.blockSignals(False)
- def _select_camera_by_index(self, index: int, fallback_text: Optional[str] = None) -> None:
- self.camera_index.blockSignals(True)
- for row in range(self.camera_index.count()):
- if self.camera_index.itemData(row) == index:
- self.camera_index.setCurrentIndex(row)
- break
- else:
- text = fallback_text if fallback_text is not None else str(index)
- self.camera_index.setEditText(text)
- self.camera_index.blockSignals(False)
-
- def _current_camera_index_value(self) -> Optional[int]:
- data = self.camera_index.currentData()
- if isinstance(data, int):
- return data
- text = self.camera_index.currentText().strip()
- if not text:
- return None
- try:
- return int(text)
- except ValueError:
- return None
-
def _parse_json(self, value: str) -> dict:
text = value.strip()
if not text:
From c5659d7f0c8fa950e49b53ae3ca0213d6e652bf6 Mon Sep 17 00:00:00 2001
From: Mackenzie Mathis
Date: Fri, 5 Dec 2025 18:15:45 +0100
Subject: [PATCH 26/26] Rename DLC Processor to DeepLabCut Processor
---
dlclivegui/processors/PLUGIN_SYSTEM.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dlclivegui/processors/PLUGIN_SYSTEM.md b/dlclivegui/processors/PLUGIN_SYSTEM.md
index fc9cab4..0c0351d 100644
--- a/dlclivegui/processors/PLUGIN_SYSTEM.md
+++ b/dlclivegui/processors/PLUGIN_SYSTEM.md
@@ -1,4 +1,4 @@
-# DLC Processor Plugin System
+# DeepLabCut Processor Plugin System
This folder contains a plugin-style architecture for DLC processors that allows GUI tools to discover and instantiate processors dynamically.