From 47c8761b97f068362acd5e71dcc15e3d745a1e08 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 21:43:16 +0300 Subject: [PATCH 01/13] feat(analytics): phase 15 core + ONNX detector (15.1/15.2) - New OpenIPC.Viewer.Analytics project on ONNX Runtime (no Python sidecar) - Core contracts: IObjectDetector, ModelSpec, Detection, FrameBuffer, ExecutionProvider - Execution-provider selection with mandatory CPU fallback (never crash on GPU) - Pure YOLOX post-processor (grid/stride decode + per-class NMS) + letterbox map - Unit tests for decode, NMS, class filter, threshold, and inverse letterbox --- Directory.Packages.props | 7 + OpenIPC.Viewer.slnx | 5 +- .../ExecutionProviderChain.cs | 30 +++ .../OnnxObjectDetector.cs | 172 ++++++++++++++++++ .../OpenIPC.Viewer.Analytics.csproj | 16 ++ .../Analytics/AiAcceleration.cs | 10 + .../Analytics/CocoClasses.cs | 24 +++ .../Analytics/DetectOptions.cs | 11 ++ .../Analytics/Detection.cs | 13 ++ .../Analytics/ExecutionProvider.cs | 16 ++ .../Analytics/FrameBuffer.cs | 22 +++ .../Analytics/IObjectDetector.cs | 22 +++ .../Analytics/LetterboxTransform.cs | 53 ++++++ .../Analytics/ModelSpec.cs | 28 +++ .../Analytics/YoloxPostProcessor.cs | 155 ++++++++++++++++ .../Analytics/LetterboxTransformTests.cs | 52 ++++++ .../Analytics/YoloxPostProcessorTests.cs | 132 ++++++++++++++ 17 files changed, 767 insertions(+), 1 deletion(-) create mode 100644 src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs create mode 100644 src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs create mode 100644 src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj create mode 100644 src/OpenIPC.Viewer.Core/Analytics/AiAcceleration.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/CocoClasses.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/DetectOptions.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/Detection.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/ExecutionProvider.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/FrameBuffer.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/IObjectDetector.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/LetterboxTransform.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/ModelSpec.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/YoloxPostProcessor.cs create mode 100644 tests/OpenIPC.Viewer.Core.Tests/Analytics/LetterboxTransformTests.cs create mode 100644 tests/OpenIPC.Viewer.Core.Tests/Analytics/YoloxPostProcessorTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index e104129..829720f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -64,6 +64,13 @@ + + + diff --git a/OpenIPC.Viewer.slnx b/OpenIPC.Viewer.slnx index 4497995..7d1a53e 100644 --- a/OpenIPC.Viewer.slnx +++ b/OpenIPC.Viewer.slnx @@ -3,6 +3,7 @@ + @@ -14,7 +15,9 @@ - + + + diff --git a/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs b/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs new file mode 100644 index 0000000..e8bda19 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// The ordered list of execution providers to try for the current platform, +// most-preferred first. CPU is always the implicit final fallback (the base +// ONNX Runtime package registers it on every RID) so it is not listed here. +// +// Only providers reachable through the generic AppendExecutionProvider(string) +// API are listed — they no-op/throw gracefully when their native EP isn't +// compiled into the referenced package, and we catch + fall back. DirectML and +// CUDA need their dedicated packages + AppendExecutionProvider_DML/_CUDA calls +// wired in the platform composition; until those are added per head, Windows +// and Linux run on CPU. +internal static class ExecutionProviderChain +{ + public static IReadOnlyList<(string Name, ExecutionProvider Provider)> ForCurrentPlatform() + { + if (OperatingSystem.IsAndroid()) + return new[] { ("NNAPI", ExecutionProvider.NnApi), ("XNNPACK", ExecutionProvider.Xnnpack) }; + + if (OperatingSystem.IsIOS() || OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst()) + return new[] { ("CoreML", ExecutionProvider.CoreMl) }; + + // Windows / Linux: CPU only for now (GPU EPs are a per-head package job). + return Array.Empty<(string, ExecutionProvider)>(); + } +} diff --git a/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs b/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs new file mode 100644 index 0000000..868fc19 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// ONNX Runtime object detector (Phase 15.1/15.2). Loads a YOLOX model, selects +// the platform execution provider with a mandatory CPU fallback, and runs +// synchronous inference (the sampling stage owns the worker thread). Not +// thread-safe across concurrent Detect calls — one session, one caller. +public sealed class OnnxObjectDetector : IObjectDetector +{ + private readonly ILogger _log; + private readonly object _gate = new(); + + private InferenceSession? _session; + private ModelSpec? _spec; + private string? _inputName; + + public OnnxObjectDetector(ILogger log) => _log = log; + + public bool IsLoaded => _session is not null; + public ExecutionProvider ActiveProvider { get; private set; } = ExecutionProvider.Cpu; + + public async Task LoadAsync(ModelSpec model, AiAcceleration acceleration, CancellationToken ct) + { + if (model is null) throw new ArgumentNullException(nameof(model)); + + var (session, provider) = await Task.Run(() => CreateSession(model, acceleration), ct) + .ConfigureAwait(false); + + lock (_gate) + { + _session?.Dispose(); + _session = session; + _spec = model; + _inputName = FirstInputName(session); + ActiveProvider = provider; + } + + _log.LogInformation("Detector loaded: {Model} on {Provider} (input {Name})", + model.Name, provider, _inputName); + } + + private (InferenceSession, ExecutionProvider) CreateSession(ModelSpec model, AiAcceleration acceleration) + { + var chain = acceleration == AiAcceleration.ForceCpu + ? Array.Empty<(string Name, ExecutionProvider Provider)>() + : ExecutionProviderChain.ForCurrentPlatform(); + + foreach (var (name, provider) in chain) + { + try + { + var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; + options.AppendExecutionProvider(name); + var session = new InferenceSession(model.FilePath, options); + return (session, provider); + } + catch (Exception ex) + { + _log.LogInformation("Execution provider {Provider} unavailable ({Error}); trying next.", + provider, ex.Message); + } + } + + // CPU is always available — this is the guaranteed fallback path. + var cpuOptions = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; + return (new InferenceSession(model.FilePath, cpuOptions), ExecutionProvider.Cpu); + } + + public IReadOnlyList Detect(FrameBuffer frame, DetectOptions options) + { + if (frame is null) throw new ArgumentNullException(nameof(frame)); + + InferenceSession session; + ModelSpec spec; + string inputName; + lock (_gate) + { + if (_session is null || _spec is null || _inputName is null) + throw new InvalidOperationException("Detector is not loaded. Call LoadAsync first."); + session = _session; + spec = _spec; + inputName = _inputName; + } + + var letterbox = new LetterboxTransform(frame.Width, frame.Height, spec.InputWidth, spec.InputHeight); + var input = BuildInputTensor(frame, letterbox, spec.InputWidth, spec.InputHeight); + + var inputs = new[] { NamedOnnxValue.CreateFromTensor(inputName, input) }; + using var results = session.Run(inputs); + var output = results[0].AsTensor().ToArray(); + + var raw = YoloxPostProcessor.Process( + output, spec.ClassCount, spec.InputWidth, spec.InputHeight, + spec.Strides, spec.GridDecodeRequired, options); + + var detections = new List(raw.Count); + foreach (var r in raw) + { + var name = r.ClassId >= 0 && r.ClassId < spec.ClassNames.Count + ? spec.ClassNames[r.ClassId] + : $"class{r.ClassId}"; + detections.Add(letterbox.MapToSource(r.ClassId, name, r.Confidence, r.X, r.Y, r.Width, r.Height)); + } + + return detections; + } + + // BGRA source -> CHW float tensor in YOLOX layout: BGR channel order, raw + // 0..255 values (no /255 — the YOLOX export folds normalization in), padded + // with 114, aspect-preserving (nearest-neighbour resize keeps this cheap). + private static DenseTensor BuildInputTensor( + FrameBuffer frame, LetterboxTransform lb, int inputW, int inputH) + { + var tensor = new DenseTensor(new[] { 1, 3, inputH, inputW }); + var buf = tensor.Buffer.Span; + buf.Fill(114f); + + var plane = inputH * inputW; + var dstX0 = (int)MathF.Round(lb.PadX); + var dstY0 = (int)MathF.Round(lb.PadY); + var scaledW = (int)MathF.Round(frame.Width * lb.Scale); + var scaledH = (int)MathF.Round(frame.Height * lb.Scale); + var invScale = 1f / lb.Scale; + + for (var dy = 0; dy < scaledH; dy++) + { + var outY = dstY0 + dy; + if (outY < 0 || outY >= inputH) continue; + var sy = Math.Min(frame.Height - 1, (int)(dy * invScale)); + + for (var dx = 0; dx < scaledW; dx++) + { + var outX = dstX0 + dx; + if (outX < 0 || outX >= inputW) continue; + var sx = Math.Min(frame.Width - 1, (int)(dx * invScale)); + + var so = sy * frame.Stride + sx * 4; + var p = outY * inputW + outX; + buf[p] = frame.Bgra[so]; // B + buf[plane + p] = frame.Bgra[so + 1]; // G + buf[2 * plane + p] = frame.Bgra[so + 2]; // R + } + } + + return tensor; + } + + private static string FirstInputName(InferenceSession session) + { + foreach (var key in session.InputMetadata.Keys) + return key; + throw new InvalidOperationException("Model has no inputs."); + } + + public ValueTask DisposeAsync() + { + lock (_gate) + { + _session?.Dispose(); + _session = null; + } + return default; + } +} diff --git a/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj new file mode 100644 index 0000000..ccd0071 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj @@ -0,0 +1,16 @@ + + + + net9.0 + + + + + + + + + + + + diff --git a/src/OpenIPC.Viewer.Core/Analytics/AiAcceleration.cs b/src/OpenIPC.Viewer.Core/Analytics/AiAcceleration.cs new file mode 100644 index 0000000..f283365 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/AiAcceleration.cs @@ -0,0 +1,10 @@ +namespace OpenIPC.Viewer.Core.Analytics; + +// Global "AI acceleration" preference (Phase 15.2). Auto lets the detector pick +// the platform-preferred execution provider with a CPU fallback; ForceCpu pins +// the CPU provider for reproducibility / troubleshooting flaky GPU stacks. +public enum AiAcceleration +{ + Auto = 0, + ForceCpu = 1, +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/CocoClasses.cs b/src/OpenIPC.Viewer.Core/Analytics/CocoClasses.cs new file mode 100644 index 0000000..f67a1d1 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/CocoClasses.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; + +namespace OpenIPC.Viewer.Core.Analytics; + +// The 80 COCO classes in the canonical training order. YOLOX (and most COCO +// detectors) emit class indices against this list, so the names map by index. +public static class CocoClasses +{ + public static readonly IReadOnlyList Names = new[] + { + "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck", + "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench", + "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra", + "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", + "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove", + "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", + "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange", + "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch", + "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse", + "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink", + "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", + "hair drier", "toothbrush", + }; +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/DetectOptions.cs b/src/OpenIPC.Viewer.Core/Analytics/DetectOptions.cs new file mode 100644 index 0000000..4e42522 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/DetectOptions.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Per-detect tuning (Phase 15.4 — sourced from per-camera settings). ClassFilter +// null means "keep all classes"; an empty set means "keep none" (analytics +// effectively idle). Thresholds cut false positives from the small models. +public sealed record DetectOptions( + float ConfidenceThreshold = 0.5f, + float NmsIouThreshold = 0.45f, + IReadOnlyCollection? ClassFilter = null); diff --git a/src/OpenIPC.Viewer.Core/Analytics/Detection.cs b/src/OpenIPC.Viewer.Core/Analytics/Detection.cs new file mode 100644 index 0000000..98c35b7 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/Detection.cs @@ -0,0 +1,13 @@ +namespace OpenIPC.Viewer.Core.Analytics; + +// One detected object. The bounding box is normalized 0..1 relative to the +// SOURCE frame (already un-letterboxed by the detector), so the overlay can +// scale it to any tile size without knowing the model input resolution. +public readonly record struct Detection( + int ClassId, + string ClassName, + float Confidence, + float X, + float Y, + float Width, + float Height); diff --git a/src/OpenIPC.Viewer.Core/Analytics/ExecutionProvider.cs b/src/OpenIPC.Viewer.Core/Analytics/ExecutionProvider.cs new file mode 100644 index 0000000..11dbf1e --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/ExecutionProvider.cs @@ -0,0 +1,16 @@ +namespace OpenIPC.Viewer.Core.Analytics; + +// Which ONNX Runtime execution provider actually backed the loaded session. +// The detector tries the platform-preferred provider and silently falls back +// to Cpu if it fails to initialize (Phase 15.2 — never crash because of GPU). +// Surfaced in the control center so the user sees what really runs. +public enum ExecutionProvider +{ + Cpu = 0, + DirectMl, // Windows + Cuda, // Linux/Windows NVIDIA + OpenVino, // Linux Intel + CoreMl, // macOS / iOS + NnApi, // Android + Xnnpack, // mobile CPU SIMD +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/FrameBuffer.cs b/src/OpenIPC.Viewer.Core/Analytics/FrameBuffer.cs new file mode 100644 index 0000000..1f5a415 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/FrameBuffer.cs @@ -0,0 +1,22 @@ +namespace OpenIPC.Viewer.Core.Analytics; + +// A single frame handed to the detector. Pixels are BGRA8888 — the same layout +// the video pipeline already produces (VideoFrame.Bgra) — so the sampling stage +// can forward decoded frames without a colour conversion. The detector does the +// letterbox + normalize to the model input internally; callers pass whatever +// resolution they have (full frame or a cheap downscale). +public sealed class FrameBuffer +{ + public FrameBuffer(byte[] bgra, int width, int height, int stride) + { + Bgra = bgra; + Width = width; + Height = height; + Stride = stride; + } + + public byte[] Bgra { get; } + public int Width { get; } + public int Height { get; } + public int Stride { get; } +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/IObjectDetector.cs b/src/OpenIPC.Viewer.Core/Analytics/IObjectDetector.cs new file mode 100644 index 0000000..450283d --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/IObjectDetector.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Cross-platform object-detection contract (Phase 15.1). The implementation +// lives in OpenIPC.Viewer.Analytics on ONNX Runtime and is wired per head via +// DI; App/Core only see this interface. Detect is synchronous and meant to run +// on a dedicated worker thread (the sampling stage owns the threading). +public interface IObjectDetector : IAsyncDisposable +{ + bool IsLoaded { get; } + + // What execution provider actually initialized (after the fallback chain). + ExecutionProvider ActiveProvider { get; } + + Task LoadAsync(ModelSpec model, AiAcceleration acceleration, CancellationToken ct); + + IReadOnlyList Detect(FrameBuffer frame, DetectOptions options); +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/LetterboxTransform.cs b/src/OpenIPC.Viewer.Core/Analytics/LetterboxTransform.cs new file mode 100644 index 0000000..281f4d8 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/LetterboxTransform.cs @@ -0,0 +1,53 @@ +using System; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Aspect-preserving letterbox from a source frame into the square model input, +// with centered padding. The detector uses Compute to know the scale/pad when +// building the input tensor, and MapToSource to turn an input-pixel box back +// into a source-normalized Detection. Pure + allocation-free so it unit-tests +// independently of ONNX Runtime (Phase 15 note: box coords must be mapped back +// through the letterbox, accounting for aspect ratio). +public readonly struct LetterboxTransform +{ + public LetterboxTransform(int sourceWidth, int sourceHeight, int inputWidth, int inputHeight) + { + SourceWidth = sourceWidth; + SourceHeight = sourceHeight; + InputWidth = inputWidth; + InputHeight = inputHeight; + Scale = Math.Min((float)inputWidth / sourceWidth, (float)inputHeight / sourceHeight); + PadX = (inputWidth - sourceWidth * Scale) / 2f; + PadY = (inputHeight - sourceHeight * Scale) / 2f; + } + + public int SourceWidth { get; } + public int SourceHeight { get; } + public int InputWidth { get; } + public int InputHeight { get; } + + // Source pixel -> input pixel; PadX/PadY are the centered black bars. + public float Scale { get; } + public float PadX { get; } + public float PadY { get; } + + // Map an input-pixel top-left box (x,y,w,h) to a source-normalized 0..1 + // Detection, clamped to the frame. + public Detection MapToSource(int classId, string className, float confidence, + float inX, float inY, float inW, float inH) + { + var sx = (inX - PadX) / Scale / SourceWidth; + var sy = (inY - PadY) / Scale / SourceHeight; + var sw = inW / Scale / SourceWidth; + var sh = inH / Scale / SourceHeight; + + // Clamp to [0,1] while keeping the box inside the frame. + var x0 = Clamp01(sx); + var y0 = Clamp01(sy); + var x1 = Clamp01(sx + sw); + var y1 = Clamp01(sy + sh); + return new Detection(classId, className, confidence, x0, y0, x1 - x0, y1 - y0); + } + + private static float Clamp01(float v) => v < 0f ? 0f : v > 1f ? 1f : v; +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/ModelSpec.cs b/src/OpenIPC.Viewer.Core/Analytics/ModelSpec.cs new file mode 100644 index 0000000..6a30384 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/ModelSpec.cs @@ -0,0 +1,28 @@ +using System.Collections.Generic; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Describes an ONNX detection model: where the file is, its input geometry, the +// class label list, and how to decode its output. Kept model-agnostic on +// purpose (Phase 15 technical notes — the post-processor is pluggable); the +// YOLOX factories below pin the values for the bundled Apache-2.0 models. +public sealed record ModelSpec( + string Name, + string FilePath, + int InputWidth, + int InputHeight, + IReadOnlyList ClassNames, + IReadOnlyList Strides, + bool GridDecodeRequired) +{ + public int ClassCount => ClassNames.Count; + + // YOLOX-tiny / YOLOX-nano: 416×416 input, FPN strides 8/16/32, the official + // ONNX export leaves box coords undecoded (grid + exp(·)·stride applied at + // post-process time), obj/cls are already sigmoid-activated. + public static ModelSpec YoloxTiny(string filePath) => + new("YOLOX-tiny", filePath, 416, 416, CocoClasses.Names, new[] { 8, 16, 32 }, GridDecodeRequired: true); + + public static ModelSpec YoloxNano(string filePath) => + new("YOLOX-nano", filePath, 416, 416, CocoClasses.Names, new[] { 8, 16, 32 }, GridDecodeRequired: true); +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/YoloxPostProcessor.cs b/src/OpenIPC.Viewer.Core/Analytics/YoloxPostProcessor.cs new file mode 100644 index 0000000..4fc3898 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/YoloxPostProcessor.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; + +namespace OpenIPC.Viewer.Core.Analytics; + +// A single decoded candidate in INPUT-pixel space (top-left x/y + w/h), before +// letterbox un-mapping. Kept separate from Detection so the post-processor +// stays pure and testable without any frame/letterbox context. +public readonly record struct RawDetection( + int ClassId, + float Confidence, + float X, + float Y, + float Width, + float Height); + +// Pure YOLOX output decoder + non-max suppression (Phase 15.1). The official +// YOLOX ONNX export emits, per anchor, [cx, cy, w, h, obj, cls0..clsN] where +// obj/cls are already sigmoid-activated and the box needs grid+stride decode +// (cx,cy = (raw+grid)*stride, w,h = exp(raw)*stride). No ONNX Runtime types +// here on purpose — feed it a flat float[] and it returns boxes, so unit tests +// can drive synthetic tensors. +public static class YoloxPostProcessor +{ + // Total anchors a square-strided model emits for the given input geometry. + public static int AnchorCount(int inputWidth, int inputHeight, IReadOnlyList strides) + { + var total = 0; + foreach (var s in strides) + total += (inputWidth / s) * (inputHeight / s); + return total; + } + + public static IReadOnlyList Process( + float[] output, + int classCount, + int inputWidth, + int inputHeight, + IReadOnlyList strides, + bool gridDecode, + DetectOptions options) + { + if (output is null) throw new ArgumentNullException(nameof(output)); + if (strides is null) throw new ArgumentNullException(nameof(strides)); + + var channels = 5 + classCount; + var anchors = AnchorCount(inputWidth, inputHeight, strides); + if (output.Length != anchors * channels) + { + throw new ArgumentException( + $"Output length {output.Length} does not match {anchors} anchors × {channels} channels.", + nameof(output)); + } + + var threshold = options.ConfidenceThreshold; + var filter = options.ClassFilter; + var candidates = new List(); + + var anchor = 0; + foreach (var stride in strides) + { + var gridW = inputWidth / stride; + var gridH = inputHeight / stride; + for (var gy = 0; gy < gridH; gy++) + { + for (var gx = 0; gx < gridW; gx++, anchor++) + { + var b = anchor * channels; + + float cx, cy, w, h; + if (gridDecode) + { + cx = (output[b + 0] + gx) * stride; + cy = (output[b + 1] + gy) * stride; + w = (float)Math.Exp(output[b + 2]) * stride; + h = (float)Math.Exp(output[b + 3]) * stride; + } + else + { + cx = output[b + 0]; + cy = output[b + 1]; + w = output[b + 2]; + h = output[b + 3]; + } + + var obj = output[b + 4]; + var bestClass = -1; + var bestProb = 0f; + for (var c = 0; c < classCount; c++) + { + var p = output[b + 5 + c]; + if (p > bestProb) + { + bestProb = p; + bestClass = c; + } + } + + var score = obj * bestProb; + if (bestClass < 0 || score < threshold) continue; + if (filter is not null && !filter.Contains(bestClass)) continue; + + candidates.Add(new RawDetection(bestClass, score, cx - w / 2f, cy - h / 2f, w, h)); + } + } + } + + return NonMaxSuppression(candidates, options.NmsIouThreshold); + } + + // Greedy per-class NMS: keep the highest-scoring box, drop same-class boxes + // that overlap it beyond the IoU threshold. + public static IReadOnlyList NonMaxSuppression( + List candidates, float iouThreshold) + { + candidates.Sort((a, b) => b.Confidence.CompareTo(a.Confidence)); + var kept = new List(); + var removed = new bool[candidates.Count]; + + for (var i = 0; i < candidates.Count; i++) + { + if (removed[i]) continue; + var a = candidates[i]; + kept.Add(a); + for (var j = i + 1; j < candidates.Count; j++) + { + if (removed[j] || candidates[j].ClassId != a.ClassId) continue; + if (Iou(a, candidates[j]) > iouThreshold) removed[j] = true; + } + } + + return kept; + } + + private static float Iou(in RawDetection a, in RawDetection b) + { + var ax2 = a.X + a.Width; + var ay2 = a.Y + a.Height; + var bx2 = b.X + b.Width; + var by2 = b.Y + b.Height; + + var ix1 = Math.Max(a.X, b.X); + var iy1 = Math.Max(a.Y, b.Y); + var ix2 = Math.Min(ax2, bx2); + var iy2 = Math.Min(ay2, by2); + + var iw = Math.Max(0f, ix2 - ix1); + var ih = Math.Max(0f, iy2 - iy1); + var inter = iw * ih; + if (inter <= 0f) return 0f; + + var union = a.Width * a.Height + b.Width * b.Height - inter; + return union <= 0f ? 0f : inter / union; + } +} diff --git a/tests/OpenIPC.Viewer.Core.Tests/Analytics/LetterboxTransformTests.cs b/tests/OpenIPC.Viewer.Core.Tests/Analytics/LetterboxTransformTests.cs new file mode 100644 index 0000000..967be13 --- /dev/null +++ b/tests/OpenIPC.Viewer.Core.Tests/Analytics/LetterboxTransformTests.cs @@ -0,0 +1,52 @@ +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Core.Tests.Analytics; + +// Letterbox geometry + inverse mapping (Phase 15.1). Boxes come out of the +// model in input-pixel space; MapToSource must undo the aspect-preserving +// scale + centered pad back into source-normalized 0..1 coords. +public sealed class LetterboxTransformTests +{ + [Fact] + public void Square_NoPadding_FullBoxMapsToWholeFrame() + { + var lb = new LetterboxTransform(100, 100, 416, 416); + Assert.Equal(4.16f, lb.Scale, 3); + Assert.Equal(0f, lb.PadX, 3); + Assert.Equal(0f, lb.PadY, 3); + + var det = lb.MapToSource(0, "person", 1f, inX: 0, inY: 0, inW: 416, inH: 416); + Assert.Equal(0f, det.X, 4); + Assert.Equal(0f, det.Y, 4); + Assert.Equal(1f, det.Width, 4); + Assert.Equal(1f, det.Height, 4); + } + + [Fact] + public void Wide_Source_PadsVertically_AndInverseMapsBack() + { + // 200×100 into 100×100: scale 0.5, scaled 100×50, vertical pad 25 each. + var lb = new LetterboxTransform(200, 100, 100, 100); + Assert.Equal(0.5f, lb.Scale, 3); + Assert.Equal(0f, lb.PadX, 3); + Assert.Equal(25f, lb.PadY, 3); + + var det = lb.MapToSource(2, "car", 0.8f, inX: 0, inY: 25, inW: 100, inH: 50); + Assert.Equal(0f, det.X, 4); + Assert.Equal(0f, det.Y, 4); + Assert.Equal(1f, det.Width, 4); + Assert.Equal(1f, det.Height, 4); + } + + [Fact] + public void MapToSource_ClampsToFrame() + { + var lb = new LetterboxTransform(100, 100, 100, 100); + // Box partly outside the frame is clamped to [0,1]. + var det = lb.MapToSource(0, "person", 1f, inX: -20, inY: -20, inW: 60, inH: 60); + Assert.Equal(0f, det.X, 4); + Assert.Equal(0f, det.Y, 4); + Assert.Equal(0.4f, det.Width, 4); // (-20+40)/100 = 0.4 + Assert.Equal(0.4f, det.Height, 4); + } +} diff --git a/tests/OpenIPC.Viewer.Core.Tests/Analytics/YoloxPostProcessorTests.cs b/tests/OpenIPC.Viewer.Core.Tests/Analytics/YoloxPostProcessorTests.cs new file mode 100644 index 0000000..904275b --- /dev/null +++ b/tests/OpenIPC.Viewer.Core.Tests/Analytics/YoloxPostProcessorTests.cs @@ -0,0 +1,132 @@ +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Core.Tests.Analytics; + +// Pure YOLOX decode + NMS + class-filter (Phase 15.1). Builds synthetic output +// tensors over a small 64×64 / strides[16,32] geometry so the maths is checkable +// by hand. Layout per anchor: [cx, cy, w, h, obj, cls0, cls1]. +public sealed class YoloxPostProcessorTests +{ + private const int Input = 64; + private static readonly int[] Strides = { 16, 32 }; + private const int Classes = 2; + private const int Channels = 5 + Classes; // 7 + + [Fact] + public void AnchorCount_MatchesYoloxTiny416() + { + // (52² + 26² + 13²) = 3549 anchors for 416 input, strides 8/16/32. + Assert.Equal(3549, YoloxPostProcessor.AnchorCount(416, 416, new[] { 8, 16, 32 })); + } + + [Fact] + public void AnchorCount_MatchesTestGeometry() + { + // 64/16 = 4 → 16, 64/32 = 2 → 4, total 20. + Assert.Equal(20, YoloxPostProcessor.AnchorCount(Input, Input, Strides)); + } + + [Fact] + public void Process_DecodesGridAndStride_ProducesExpectedBox() + { + var output = NewOutput(); + // stride-16 block, grid (gx=2, gy=1) → anchor index 1*4 + 2 = 6. + SetAnchor(output, index: 6, rawX: 0.5f, rawY: 0.5f, rawW: 0f, rawH: 0f, + obj: 1f, classProbs: (0.9f, 0f)); + + var result = YoloxPostProcessor.Process( + output, Classes, Input, Input, Strides, gridDecode: true, + new DetectOptions(ConfidenceThreshold: 0.5f)); + + var det = Assert.Single(result); + Assert.Equal(0, det.ClassId); + Assert.Equal(0.9f, det.Confidence, 3); + // cx=(0.5+2)*16=40, cy=(0.5+1)*16=24, w=h=exp(0)*16=16 → top-left (32,16). + Assert.Equal(32f, det.X, 2); + Assert.Equal(16f, det.Y, 2); + Assert.Equal(16f, det.Width, 2); + Assert.Equal(16f, det.Height, 2); + } + + [Fact] + public void Process_ClassFilter_DropsUnselectedClass() + { + var output = NewOutput(); + SetAnchor(output, index: 6, 0.5f, 0.5f, 0f, 0f, obj: 1f, classProbs: (0.9f, 0f)); + + var result = YoloxPostProcessor.Process( + output, Classes, Input, Input, Strides, gridDecode: true, + new DetectOptions(ConfidenceThreshold: 0.5f, ClassFilter: new[] { 1 })); + + Assert.Empty(result); // best class is 0, filter only keeps class 1 + } + + [Fact] + public void Process_BelowThreshold_IsDropped() + { + var output = NewOutput(); + SetAnchor(output, index: 6, 0.5f, 0.5f, 0f, 0f, obj: 0.5f, classProbs: (0.4f, 0f)); // score 0.2 + + var result = YoloxPostProcessor.Process( + output, Classes, Input, Input, Strides, gridDecode: true, + new DetectOptions(ConfidenceThreshold: 0.5f)); + + Assert.Empty(result); + } + + [Fact] + public void Process_Nms_SuppressesOverlappingSameClass() + { + var output = NewOutput(); + // Two anchors decoding to the same box, same class, different scores. + SetAnchor(output, index: 6, rawX: 0.5f, rawY: 0.5f, rawW: 0f, rawH: 0f, + obj: 1f, classProbs: (0.9f, 0f)); // cx40 cy24 + // grid (gx=3, gy=1) → anchor 7; rawX=-0.5 → cx=(-0.5+3)*16=40, same box. + SetAnchor(output, index: 7, rawX: -0.5f, rawY: 0.5f, rawW: 0f, rawH: 0f, + obj: 1f, classProbs: (0.8f, 0f)); + + var result = YoloxPostProcessor.Process( + output, Classes, Input, Input, Strides, gridDecode: true, + new DetectOptions(ConfidenceThreshold: 0.5f, NmsIouThreshold: 0.45f)); + + var det = Assert.Single(result); + Assert.Equal(0.9f, det.Confidence, 3); // the higher-scoring box survives + } + + [Fact] + public void Process_DifferentClasses_AreNotSuppressed() + { + var output = NewOutput(); + SetAnchor(output, index: 6, 0.5f, 0.5f, 0f, 0f, obj: 1f, classProbs: (0.9f, 0f)); // class0 + SetAnchor(output, index: 7, -0.5f, 0.5f, 0f, 0f, obj: 1f, classProbs: (0f, 0.85f)); // class1, same box + + var result = YoloxPostProcessor.Process( + output, Classes, Input, Input, Strides, gridDecode: true, + new DetectOptions(ConfidenceThreshold: 0.5f, NmsIouThreshold: 0.45f)); + + Assert.Equal(2, result.Count); + } + + [Fact] + public void Process_RejectsMismatchedOutputLength() + { + Assert.Throws(() => YoloxPostProcessor.Process( + new float[5], Classes, Input, Input, Strides, gridDecode: true, new DetectOptions())); + } + + private static float[] NewOutput() => + new float[YoloxPostProcessor.AnchorCount(Input, Input, Strides) * Channels]; + + private static void SetAnchor(float[] output, int index, + float rawX, float rawY, float rawW, float rawH, float obj, (float c0, float c1) classProbs) + { + var b = index * Channels; + output[b + 0] = rawX; + output[b + 1] = rawY; + output[b + 2] = rawW; + output[b + 3] = rawH; + output[b + 4] = obj; + output[b + 5] = classProbs.c0; + output[b + 6] = classProbs.c1; + } +} From 25417f13bc2c205f995407298b7a9b6962859639 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 21:49:45 +0300 Subject: [PATCH 02/13] feat(analytics): sampling pipeline + model delivery (15.3) - IAnalyticsEngine + ObjectDetectionEngine: shared detector, one inference worker - FrameSampler thins decoder FPS to analyticsFps; bounded drop-oldest channel - Full-frame downscale-copy off the decoder thread (buffer is session-owned) - IModelProvider/ModelProvider: download-on-first-enable, AppData cache, SHA-256 - ModelCatalog pins Apache-2.0 YOLOX-tiny; AnalyticsSettings + diagnostics - FrameSampler + settings unit tests --- src/OpenIPC.Viewer.Analytics/ModelCatalog.cs | 31 +++ src/OpenIPC.Viewer.Analytics/ModelProvider.cs | 109 ++++++++ .../ObjectDetectionEngine.cs | 253 ++++++++++++++++++ .../OpenIPC.Viewer.Analytics.csproj | 1 + .../Analytics/AnalyticsDiagnostics.cs | 16 ++ .../Analytics/AnalyticsSettings.cs | 23 ++ .../Analytics/DetectionResult.cs | 14 + .../Analytics/FrameSampler.cs | 35 +++ .../Analytics/IAnalyticsEngine.cs | 31 +++ .../Analytics/IModelProvider.cs | 13 + .../Analytics/FrameSamplerTests.cs | 64 +++++ 11 files changed, 590 insertions(+) create mode 100644 src/OpenIPC.Viewer.Analytics/ModelCatalog.cs create mode 100644 src/OpenIPC.Viewer.Analytics/ModelProvider.cs create mode 100644 src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/AnalyticsDiagnostics.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/AnalyticsSettings.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/DetectionResult.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/FrameSampler.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/IAnalyticsEngine.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/IModelProvider.cs create mode 100644 tests/OpenIPC.Viewer.Core.Tests/Analytics/FrameSamplerTests.cs diff --git a/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs b/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs new file mode 100644 index 0000000..7b991c3 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs @@ -0,0 +1,31 @@ +using System; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// A downloadable detection model: where to fetch it, its integrity hash, and +// how to build the ModelSpec once it is on disk. +public sealed record ModelDescriptor( + string Name, + string FileName, + Uri? DownloadUri, + string? Sha256Hex, + Func CreateSpec); + +// Known models. We ship Apache-2.0 YOLOX (never AGPL Ultralytics YOLO) to stay +// MIT/store compatible. The model is an asset fetched on first enable, not +// checked into the repo. +public static class ModelCatalog +{ + // Official YOLOX-tiny ONNX export from the upstream release assets. + // TODO: pin Sha256Hex once the asset is downloaded and verified in CI; + // until then integrity checking is skipped (logged). + public static ModelDescriptor YoloxTiny { get; } = new( + Name: "YOLOX-tiny", + FileName: "yolox_tiny.onnx", + DownloadUri: new Uri("https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx"), + Sha256Hex: null, + CreateSpec: ModelSpec.YoloxTiny); + + public static ModelDescriptor Default => YoloxTiny; +} diff --git a/src/OpenIPC.Viewer.Analytics/ModelProvider.cs b/src/OpenIPC.Viewer.Analytics/ModelProvider.cs new file mode 100644 index 0000000..b6454cd --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ModelProvider.cs @@ -0,0 +1,109 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Platform; + +namespace OpenIPC.Viewer.Analytics; + +// Resolves the detection model file (Phase 15 — download on first enable, cache +// in AppData, verify integrity, repo stays binary-free). Resolution order: +// 1. OPENIPC_DETECTION_MODEL env var (explicit local override / CI fixture), +// 2. cached file in {AppData}/models (re-verified if a hash is pinned), +// 3. download from the descriptor URI into the cache. +public sealed class ModelProvider : IModelProvider +{ + private readonly IFileSystem _fs; + private readonly ILogger _log; + private readonly ModelDescriptor _descriptor; + private readonly Func _httpFactory; + private readonly SemaphoreSlim _gate = new(1, 1); + + public ModelProvider(IFileSystem fs, ILogger log) + : this(fs, log, ModelCatalog.Default, () => new HttpClient { Timeout = TimeSpan.FromMinutes(5) }) + { + } + + internal ModelProvider(IFileSystem fs, ILogger log, + ModelDescriptor descriptor, Func httpFactory) + { + _fs = fs; + _log = log; + _descriptor = descriptor; + _httpFactory = httpFactory; + } + + public async Task EnsureModelAsync(CancellationToken ct) + { + var overridePath = Environment.GetEnvironmentVariable("OPENIPC_DETECTION_MODEL"); + if (!string.IsNullOrWhiteSpace(overridePath) && File.Exists(overridePath)) + { + _log.LogInformation("Using detection model override: {Path}", overridePath); + return _descriptor.CreateSpec(overridePath); + } + + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + var modelsDir = Directory.CreateDirectory(Path.Combine(_fs.AppDataDir.FullName, "models")); + var target = Path.Combine(modelsDir.FullName, _descriptor.FileName); + + if (File.Exists(target) && await VerifyAsync(target, ct).ConfigureAwait(false)) + return _descriptor.CreateSpec(target); + + if (_descriptor.DownloadUri is null) + throw new InvalidOperationException( + $"Model {_descriptor.Name} is missing and no download URI is configured."); + + await DownloadAsync(_descriptor.DownloadUri, target, ct).ConfigureAwait(false); + + if (!await VerifyAsync(target, ct).ConfigureAwait(false)) + { + File.Delete(target); + throw new InvalidOperationException( + $"Downloaded model {_descriptor.Name} failed its integrity check."); + } + + return _descriptor.CreateSpec(target); + } + finally + { + _gate.Release(); + } + } + + private async Task VerifyAsync(string path, CancellationToken ct) + { + if (_descriptor.Sha256Hex is null) return true; // not pinned yet + using var sha = SHA256.Create(); + await using var stream = File.OpenRead(path); + var hash = await sha.ComputeHashAsync(stream, ct).ConfigureAwait(false); + var hex = Convert.ToHexString(hash); + var ok = string.Equals(hex, _descriptor.Sha256Hex, StringComparison.OrdinalIgnoreCase); + if (!ok) + _log.LogWarning("Model {Name} sha256 mismatch (expected {Expected}, got {Actual}).", + _descriptor.Name, _descriptor.Sha256Hex, hex); + return ok; + } + + private async Task DownloadAsync(Uri uri, string target, CancellationToken ct) + { + _log.LogInformation("Downloading detection model {Name} from {Uri}", _descriptor.Name, uri); + using var http = _httpFactory(); + using var resp = await http.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + resp.EnsureSuccessStatusCode(); + + var tmp = target + ".part"; + await using (var dst = File.Create(tmp)) + await using (var src = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false)) + { + await src.CopyToAsync(dst, ct).ConfigureAwait(false); + } + File.Move(tmp, target, overwrite: true); + _log.LogInformation("Detection model {Name} cached at {Path}", _descriptor.Name, target); + } +} diff --git a/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs b/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs new file mode 100644 index 0000000..7dbb022 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Video; +using ReactiveSubject = System.Reactive.Subjects.Subject; + +namespace OpenIPC.Viewer.Analytics; + +// The Phase 15.3 sampling pipeline. Per camera: a FrameSampler thins the +// decoder's ~25 FPS down to analyticsFps, each kept frame is downscaled + +// copied (the decoder owns its buffer) and pushed into a small bounded channel +// with drop-oldest semantics. A single worker drains the channel and runs the +// shared detector, so memory/CPU stay bounded regardless of camera count. +public sealed class ObjectDetectionEngine : IAnalyticsEngine +{ + private const int QueueCapacity = 8; + private const int DownscaleMaxSide = 640; + + private readonly IObjectDetector _detector; + private readonly IModelProvider _modelProvider; + private readonly ILogger _log; + + private readonly ConcurrentDictionary _cameras = new(); + private readonly Channel _queue = Channel.CreateBounded( + new BoundedChannelOptions(QueueCapacity) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + }); + private readonly ReactiveSubject _results = new(); + private readonly CancellationTokenSource _shutdown = new(); + private readonly object _statsLock = new(); + private readonly SemaphoreSlim _initGate = new(1, 1); + + private long _sampled; + private long _processed; + private long _latencyCount; + private double _latencySumMs; + private Task? _worker; + private bool _initialized; + + public ObjectDetectionEngine(IObjectDetector detector, IModelProvider modelProvider, + ILogger log) + { + _detector = detector; + _modelProvider = modelProvider; + _log = log; + } + + public bool IsReady => _detector.IsLoaded; + public ExecutionProvider ActiveProvider => _detector.ActiveProvider; + public IObservable Results => _results; + + public AnalyticsDiagnostics Diagnostics + { + get + { + long sampled = Interlocked.Read(ref _sampled); + long processed = Interlocked.Read(ref _processed); + double avg; + lock (_statsLock) + avg = _latencyCount == 0 ? 0 : _latencySumMs / _latencyCount; + return new AnalyticsDiagnostics( + ActiveCameras: _cameras.Count, + FramesSampled: sampled, + FramesProcessed: processed, + FramesDropped: Math.Max(0, sampled - processed - QueueDepth()), + QueueDepth: QueueDepth(), + AverageLatencyMs: avg); + } + } + + public async Task InitializeAsync(AiAcceleration acceleration, CancellationToken ct) + { + await _initGate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_initialized) return; + var spec = await _modelProvider.EnsureModelAsync(ct).ConfigureAwait(false); + await _detector.LoadAsync(spec, acceleration, ct).ConfigureAwait(false); + _worker = Task.Run(() => WorkerLoopAsync(_shutdown.Token)); + _initialized = true; + _log.LogInformation("Analytics engine ready on {Provider}.", _detector.ActiveProvider); + } + finally + { + _initGate.Release(); + } + } + + public void Attach(CameraId cameraId, IObservable frames, + Func settings, Func isActive) + { + Detach(cameraId); + var reg = new CameraRegistration(new FrameSampler(settings().AnalyticsFps), settings, isActive); + reg.Subscription = frames.Subscribe(new FrameObserver(this, cameraId, reg)); + _cameras[cameraId] = reg; + } + + public void Detach(CameraId cameraId) + { + if (_cameras.TryRemove(cameraId, out var reg)) + reg.Dispose(); + } + + private void OnFrame(CameraId cameraId, CameraRegistration reg, in VideoFrame frame) + { + var settings = reg.Settings(); + if (!settings.Enabled || !reg.IsActive()) return; + + // Keep the sampler in step with a live FPS change. + if (reg.Sampler.TargetFps != ClampFps(settings.AnalyticsFps)) + reg.Sampler = new FrameSampler(settings.AnalyticsFps); + + if (!reg.Sampler.ShouldSample(frame.ReceivedAt)) return; + + var buffer = Downscale(frame, DownscaleMaxSide); + if (_queue.Writer.TryWrite(new WorkItem(cameraId, buffer, settings, frame.ReceivedAt))) + Interlocked.Increment(ref _sampled); + } + + private async Task WorkerLoopAsync(CancellationToken ct) + { + try + { + await foreach (var item in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false)) + { + if (!_detector.IsLoaded) continue; + + try + { + var sw = Stopwatch.StartNew(); + var detections = _detector.Detect(item.Frame, item.Settings.ToDetectOptions()); + sw.Stop(); + + Interlocked.Increment(ref _processed); + lock (_statsLock) + { + _latencySumMs += sw.Elapsed.TotalMilliseconds; + _latencyCount++; + } + + _results.OnNext(new DetectionResult( + item.CameraId, item.Timestamp, detections, sw.Elapsed.TotalMilliseconds)); + } + catch (Exception ex) + { + _log.LogWarning(ex, "Inference failed for camera {Camera}.", item.CameraId); + } + } + } + catch (OperationCanceledException) + { + // shutting down + } + } + + private int QueueDepth() => _queue.Reader.Count; + + private static int ClampFps(int fps) => fps < 1 ? 1 : fps > 30 ? 30 : fps; + + // Full-frame aspect-preserving downscale into a tightly packed BGRA buffer. + // Because it is the WHOLE frame (no crop), detection coords normalized to + // this buffer equal those of the original — the overlay maps cleanly. + private static FrameBuffer Downscale(in VideoFrame f, int maxSide) + { + var scale = Math.Min(1f, (float)maxSide / Math.Max(f.Width, f.Height)); + var w = Math.Max(1, (int)(f.Width * scale)); + var h = Math.Max(1, (int)(f.Height * scale)); + var dst = new byte[w * h * 4]; + var sxStep = (float)f.Width / w; + var syStep = (float)f.Height / h; + + for (var y = 0; y < h; y++) + { + var srcRow = Math.Min(f.Height - 1, (int)(y * syStep)) * f.Stride; + var dstRow = y * w * 4; + for (var x = 0; x < w; x++) + { + var so = srcRow + Math.Min(f.Width - 1, (int)(x * sxStep)) * 4; + var doff = dstRow + x * 4; + dst[doff] = f.Bgra[so]; + dst[doff + 1] = f.Bgra[so + 1]; + dst[doff + 2] = f.Bgra[so + 2]; + dst[doff + 3] = 255; + } + } + + return new FrameBuffer(dst, w, h, w * 4); + } + + public async ValueTask DisposeAsync() + { + _shutdown.Cancel(); + _queue.Writer.TryComplete(); + foreach (var reg in _cameras.Values) reg.Dispose(); + _cameras.Clear(); + if (_worker is not null) + { + try { await _worker.ConfigureAwait(false); } + catch (OperationCanceledException) { } + } + _results.OnCompleted(); + _results.Dispose(); + await _detector.DisposeAsync().ConfigureAwait(false); + _shutdown.Dispose(); + } + + private sealed record WorkItem(CameraId CameraId, FrameBuffer Frame, AnalyticsSettings Settings, DateTime Timestamp); + + private sealed class CameraRegistration : IDisposable + { + public CameraRegistration(FrameSampler sampler, Func settings, Func isActive) + { + Sampler = sampler; + Settings = settings; + IsActive = isActive; + } + + public FrameSampler Sampler { get; set; } + public Func Settings { get; } + public Func IsActive { get; } + public IDisposable? Subscription { get; set; } + + public void Dispose() => Subscription?.Dispose(); + } + + // Lightweight IObserver so we don't depend on Rx subscription extensions. + private sealed class FrameObserver : IObserver + { + private readonly ObjectDetectionEngine _engine; + private readonly CameraId _cameraId; + private readonly CameraRegistration _reg; + + public FrameObserver(ObjectDetectionEngine engine, CameraId cameraId, CameraRegistration reg) + { + _engine = engine; + _cameraId = cameraId; + _reg = reg; + } + + public void OnNext(VideoFrame value) => _engine.OnFrame(_cameraId, _reg, value); + public void OnError(Exception error) { } + public void OnCompleted() { } + } +} diff --git a/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj index ccd0071..2935bf3 100644 --- a/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj +++ b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj @@ -11,6 +11,7 @@ + diff --git a/src/OpenIPC.Viewer.Core/Analytics/AnalyticsDiagnostics.cs b/src/OpenIPC.Viewer.Core/Analytics/AnalyticsDiagnostics.cs new file mode 100644 index 0000000..a284a2c --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/AnalyticsDiagnostics.cs @@ -0,0 +1,16 @@ +namespace OpenIPC.Viewer.Core.Analytics; + +// Snapshot of engine health for the control center Diagnostics section +// (Phase 15.7). FramesDropped surfaces the bounded-queue back-pressure that was +// the dashboard's memory-leak trap — a steady non-zero drop count is expected +// under load; a growing queue depth is the warning sign. +public sealed record AnalyticsDiagnostics( + int ActiveCameras, + long FramesSampled, + long FramesProcessed, + long FramesDropped, + int QueueDepth, + double AverageLatencyMs) +{ + public static AnalyticsDiagnostics Empty { get; } = new(0, 0, 0, 0, 0, 0); +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/AnalyticsSettings.cs b/src/OpenIPC.Viewer.Core/Analytics/AnalyticsSettings.cs new file mode 100644 index 0000000..bb1449e --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/AnalyticsSettings.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Per-camera analytics configuration (Phase 15.4), resolved from persisted +// camera fields. ClassIds empty means "all classes" (the filter is a way to +// cut false positives, not a kill switch) — the editor pre-selects sensible +// classes like person/car so the empty case is rare. +public sealed record AnalyticsSettings( + bool Enabled = false, + IReadOnlyCollection? ClassIds = null, + float ConfidenceThreshold = 0.5f, + int AnalyticsFps = 3, + bool AutoRecord = false, + int PostEventSeconds = 15) +{ + public DetectOptions ToDetectOptions() => new( + ConfidenceThreshold, + NmsIouThreshold: 0.45f, + ClassFilter: ClassIds is { Count: > 0 } ? ClassIds : null); + + public static AnalyticsSettings Disabled { get; } = new(); +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/DetectionResult.cs b/src/OpenIPC.Viewer.Core/Analytics/DetectionResult.cs new file mode 100644 index 0000000..bcfa1bd --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/DetectionResult.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using OpenIPC.Viewer.Core.Entities; + +namespace OpenIPC.Viewer.Core.Analytics; + +// One inference outcome for a camera (Phase 15.3). Carries the detections (for +// the overlay + counter), when it happened, and how long inference took (for +// the control center latency readout). +public sealed record DetectionResult( + CameraId CameraId, + DateTime OccurredAt, + IReadOnlyList Detections, + double InferenceMs); diff --git a/src/OpenIPC.Viewer.Core/Analytics/FrameSampler.cs b/src/OpenIPC.Viewer.Core/Analytics/FrameSampler.cs new file mode 100644 index 0000000..24241dd --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/FrameSampler.cs @@ -0,0 +1,35 @@ +using System; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Rate gate for the analytics tap (Phase 15.3). The decoder produces ~25-30 +// FPS; feeding every frame to the detector is the dashboard's "8 GB in 3 hours" +// trap. ShouldSample admits at most targetFps frames per second by wall-clock +// spacing. Pure (time passed in) so it unit-tests deterministically. +public sealed class FrameSampler +{ + private readonly TimeSpan _minInterval; + private DateTime _lastAdmitted = DateTime.MinValue; + + public FrameSampler(int targetFps) + { + if (targetFps < 1) targetFps = 1; + if (targetFps > 30) targetFps = 30; + TargetFps = targetFps; + _minInterval = TimeSpan.FromSeconds(1.0 / targetFps); + } + + public int TargetFps { get; } + + public bool ShouldSample(DateTime nowUtc) + { + if (_lastAdmitted == DateTime.MinValue || nowUtc - _lastAdmitted >= _minInterval) + { + _lastAdmitted = nowUtc; + return true; + } + return false; + } + + public void Reset() => _lastAdmitted = DateTime.MinValue; +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/IAnalyticsEngine.cs b/src/OpenIPC.Viewer.Core/Analytics/IAnalyticsEngine.cs new file mode 100644 index 0000000..de87120 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/IAnalyticsEngine.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Video; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Orchestrates per-camera object detection (Phase 15.3). One shared detector +// behind a bounded queue feeds a single inference worker, so total memory and +// CPU stay bounded no matter how many cameras attach. App talks to this Core +// interface; the ONNX-backed implementation is wired per head via DI. +public interface IAnalyticsEngine : IAsyncDisposable +{ + bool IsReady { get; } + ExecutionProvider ActiveProvider { get; } + + IObservable Results { get; } + AnalyticsDiagnostics Diagnostics { get; } + + // Loads the model + detector. Safe to call once; subsequent calls no-op. + Task InitializeAsync(AiAcceleration acceleration, CancellationToken ct); + + // Begin sampling frames for a camera. settings is re-read per frame so the + // user can toggle classes/threshold live; isActive lets Smart Pause + // (Phase 12) gate analytics for hidden/suspended tiles. + void Attach(CameraId cameraId, IObservable frames, + Func settings, Func isActive); + + void Detach(CameraId cameraId); +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/IModelProvider.cs b/src/OpenIPC.Viewer.Core/Analytics/IModelProvider.cs new file mode 100644 index 0000000..8dea8fa --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/IModelProvider.cs @@ -0,0 +1,13 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Resolves the detection model to a ready-to-load ModelSpec (Phase 15 — model +// is an asset, not code; fetched on first enable). The implementation caches in +// AppData, verifies integrity, and supports a local override path so the repo +// ships no binaries. +public interface IModelProvider +{ + Task EnsureModelAsync(CancellationToken ct); +} diff --git a/tests/OpenIPC.Viewer.Core.Tests/Analytics/FrameSamplerTests.cs b/tests/OpenIPC.Viewer.Core.Tests/Analytics/FrameSamplerTests.cs new file mode 100644 index 0000000..ebb97fe --- /dev/null +++ b/tests/OpenIPC.Viewer.Core.Tests/Analytics/FrameSamplerTests.cs @@ -0,0 +1,64 @@ +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Core.Tests.Analytics; + +// Wall-clock rate gate for the analytics tap (Phase 15.3). +public sealed class FrameSamplerTests +{ + private static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + [Fact] + public void FirstFrame_IsAlwaysAdmitted() + { + var sampler = new FrameSampler(3); + Assert.True(sampler.ShouldSample(T0)); + } + + [Fact] + public void AdmitsAtTargetRate_DropsFramesInBetween() + { + var sampler = new FrameSampler(3); // one frame every ~333ms + Assert.True(sampler.ShouldSample(T0)); + Assert.False(sampler.ShouldSample(T0.AddMilliseconds(100))); + Assert.False(sampler.ShouldSample(T0.AddMilliseconds(300))); + Assert.True(sampler.ShouldSample(T0.AddMilliseconds(334))); + Assert.False(sampler.ShouldSample(T0.AddMilliseconds(400))); + Assert.True(sampler.ShouldSample(T0.AddMilliseconds(700))); + } + + [Theory] + [InlineData(0, 1)] // clamped up to 1 fps + [InlineData(-5, 1)] + [InlineData(100, 30)] // clamped down to 30 fps + [InlineData(3, 3)] + public void ClampsTargetFpsToSaneRange(int requested, int expected) + { + Assert.Equal(expected, new FrameSampler(requested).TargetFps); + } + + [Fact] + public void Reset_ReadmitsImmediately() + { + var sampler = new FrameSampler(1); + Assert.True(sampler.ShouldSample(T0)); + Assert.False(sampler.ShouldSample(T0.AddMilliseconds(500))); + sampler.Reset(); + Assert.True(sampler.ShouldSample(T0.AddMilliseconds(600))); + } + + [Fact] + public void EmptyClassFilter_MapsToAllClasses() + { + var settings = new AnalyticsSettings(Enabled: true, ClassIds: System.Array.Empty()); + Assert.Null(settings.ToDetectOptions().ClassFilter); + } + + [Fact] + public void SelectedClasses_FlowToDetectOptions() + { + var settings = new AnalyticsSettings(Enabled: true, ClassIds: new[] { 0, 2 }, ConfidenceThreshold: 0.6f); + var opts = settings.ToDetectOptions(); + Assert.Equal(0.6f, opts.ConfidenceThreshold, 3); + Assert.Equal(new[] { 0, 2 }, opts.ClassFilter); + } +} From 818d08a81d831bc36a3c888b9ad8b6d6d1c276d7 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 21:57:33 +0300 Subject: [PATCH 03/13] feat(analytics): per-camera persistence + DI wiring (15.4) - Migration 010: Ai* columns on Cameras (enabled, classes CSV, threshold, fps, auto-record) - Camera/New/UpdateCameraRequest carry AnalyticsSettings; repo maps flat columns - CameraDirectoryService.SetAnalyticsAsync + thread analytics through add/update - Global AiAcceleration user setting (auto/force-cpu) via IUserSettingsAccessor - Register IModelProvider/IObjectDetector/IAnalyticsEngine (boot-safe, opt-in) - Suppress Android XA0141 (ORT native 16KB page-size advisory, upstream) --- .../OpenIPC.Viewer.Android.csproj | 9 +++- .../Services/UserSettings.cs | 5 +- .../Services/UserSettingsService.cs | 4 ++ .../OpenIPC.Viewer.Composition.csproj | 1 + .../SharedComposition.cs | 11 ++++ src/OpenIPC.Viewer.Core/Entities/Camera.cs | 9 +++- .../Entities/NewCameraRequest.cs | 4 +- .../Entities/UpdateCameraRequest.cs | 5 +- .../Services/CameraDirectoryService.cs | 18 ++++++- .../Settings/IUserSettingsAccessor.cs | 5 ++ .../Migrations/010_ai_analytics.sql | 9 ++++ .../Persistence/SqliteCameraRepository.cs | 52 +++++++++++++++++-- 12 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 src/OpenIPC.Viewer.Infrastructure/Persistence/Migrations/010_ai_analytics.sql diff --git a/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj b/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj index d5f28e5..b2c8e88 100644 --- a/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj +++ b/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj @@ -27,8 +27,15 @@ verification, which we don't perform — credentials are AES-GCM via EncryptedFileSecretsStore, RTSP auth is HTTP digest. Revisit when MS ships a patched servicing release. + + XA0141 suppression: ONNX Runtime's android native (libonnxruntime4j_jni.so, + pulled transitively by the Phase 15 analytics engine) isn't yet aligned to + the 16 KB page size Android 16 will require. It's an upstream packaging + advisory, not our code; on-device analytics on Android is a later polish + item (capability-flagged, desktop-first). Revisit when ORT ships aligned + mobile binaries. --> - $(NoWarn);NU1903 + $(NoWarn);NU1903;XA0141 diff --git a/src/OpenIPC.Viewer.App/Services/UserSettings.cs b/src/OpenIPC.Viewer.App/Services/UserSettings.cs index 99fdf89..b05fa58 100644 --- a/src/OpenIPC.Viewer.App/Services/UserSettings.cs +++ b/src/OpenIPC.Viewer.App/Services/UserSettings.cs @@ -32,7 +32,10 @@ public sealed record UserSettings( bool SshStrictHostKey = true, int SshDefaultPort = 22, int SshTerminalFontSize = 14, - string MajesticConfigPath = "/etc/majestic.yaml") + string MajesticConfigPath = "/etc/majestic.yaml", + // Local AI analytics (Phase 15.2). "auto" lets the detector pick the + // platform execution provider with a CPU fallback; "force-cpu" pins CPU. + string AiAcceleration = "auto") { public static UserSettings Default => new(); } diff --git a/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs b/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs index 7bcfeba..66cacf8 100644 --- a/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs +++ b/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs @@ -28,6 +28,10 @@ public sealed class UserSettingsService : IUserSettingsAccessor public int SshDefaultPort => Current.SshDefaultPort < 1 ? 22 : Current.SshDefaultPort; public string MajesticConfigPath => string.IsNullOrWhiteSpace(Current.MajesticConfigPath) ? "/etc/majestic.yaml" : Current.MajesticConfigPath; + public OpenIPC.Viewer.Core.Analytics.AiAcceleration AiAcceleration => + string.Equals(Current.AiAcceleration, "force-cpu", StringComparison.OrdinalIgnoreCase) + ? OpenIPC.Viewer.Core.Analytics.AiAcceleration.ForceCpu + : OpenIPC.Viewer.Core.Analytics.AiAcceleration.Auto; private static readonly JsonSerializerOptions JsonOpts = new() { diff --git a/src/OpenIPC.Viewer.Composition/OpenIPC.Viewer.Composition.csproj b/src/OpenIPC.Viewer.Composition/OpenIPC.Viewer.Composition.csproj index 1842e2b..577d474 100644 --- a/src/OpenIPC.Viewer.Composition/OpenIPC.Viewer.Composition.csproj +++ b/src/OpenIPC.Viewer.Composition/OpenIPC.Viewer.Composition.csproj @@ -10,6 +10,7 @@ + diff --git a/src/OpenIPC.Viewer.Composition/SharedComposition.cs b/src/OpenIPC.Viewer.Composition/SharedComposition.cs index f63353d..7aa4965 100644 --- a/src/OpenIPC.Viewer.Composition/SharedComposition.cs +++ b/src/OpenIPC.Viewer.Composition/SharedComposition.cs @@ -73,6 +73,17 @@ public static IServiceCollection AddSharedServices(this IServiceCollection servi services.AddSingleton(); services.AddSingleton(); + // Local AI analytics (Phase 15). One shared detector + engine; the model + // is fetched on first enable and inference falls back to CPU when a GPU + // EP is unavailable, so registering this on every head is boot-safe + // (analytics is opt-in per camera and only initializes when enabled). + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + // Recording lifecycle (IRecorder itself is registered by the platform // host — FFmpeg subprocess on desktop, FFmpegKit on Android, etc). services.AddSingleton(); diff --git a/src/OpenIPC.Viewer.Core/Entities/Camera.cs b/src/OpenIPC.Viewer.Core/Entities/Camera.cs index dd451cc..b3929fb 100644 --- a/src/OpenIPC.Viewer.Core/Entities/Camera.cs +++ b/src/OpenIPC.Viewer.Core/Entities/Camera.cs @@ -1,4 +1,5 @@ using System; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Ssh; using OpenIPC.Viewer.Core.Video; @@ -30,8 +31,14 @@ public sealed record Camera( StreamQualityOverride StreamQualityOverride = StreamQualityOverride.Auto, // SSH port for the device suite (Phase 13). Null → default 22. Credentials // live in the secrets store under cam:{id}:ssh:* keys, not on the entity. - int? SshPort = null) + int? SshPort = null, + // Per-camera AI object detection (Phase 15). Disabled by default; stored + // flat across the Ai* columns and recomposed here. + AnalyticsSettings? Analytics = null) { + /// The camera's analytics config, never null (defaults to disabled). + public AnalyticsSettings AnalyticsOrDefault => Analytics ?? AnalyticsSettings.Disabled; + /// The SSH port to connect to, defaulting to 22 when unset. public int SshPortOrDefault => SshPort ?? SshEndpoint.DefaultPort; diff --git a/src/OpenIPC.Viewer.Core/Entities/NewCameraRequest.cs b/src/OpenIPC.Viewer.Core/Entities/NewCameraRequest.cs index 2b99fd1..b2b1711 100644 --- a/src/OpenIPC.Viewer.Core/Entities/NewCameraRequest.cs +++ b/src/OpenIPC.Viewer.Core/Entities/NewCameraRequest.cs @@ -1,4 +1,5 @@ using System; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Video; namespace OpenIPC.Viewer.Core.Entities; @@ -17,4 +18,5 @@ public sealed record NewCameraRequest( // the SSH login may differ. Null SshCredentials means "no SSH-specific // login" (the resolver falls back to the main credentials). CameraCredentials? SshCredentials = null, - int? SshPort = null); + int? SshPort = null, + AnalyticsSettings? Analytics = null); diff --git a/src/OpenIPC.Viewer.Core/Entities/UpdateCameraRequest.cs b/src/OpenIPC.Viewer.Core/Entities/UpdateCameraRequest.cs index a8b0383..c8d2434 100644 --- a/src/OpenIPC.Viewer.Core/Entities/UpdateCameraRequest.cs +++ b/src/OpenIPC.Viewer.Core/Entities/UpdateCameraRequest.cs @@ -1,4 +1,5 @@ using System; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Video; namespace OpenIPC.Viewer.Core.Entities; @@ -16,4 +17,6 @@ public sealed record UpdateCameraRequest( // SSH device suite (Phase 13). Null SshCredentials keeps the stored SSH // login untouched (mirrors how null Credentials keeps the main login). CameraCredentials? SshCredentials = null, - int? SshPort = null); + int? SshPort = null, + // Null keeps the stored analytics config untouched. + AnalyticsSettings? Analytics = null); diff --git a/src/OpenIPC.Viewer.Core/Services/CameraDirectoryService.cs b/src/OpenIPC.Viewer.Core/Services/CameraDirectoryService.cs index f15ce38..00b7b29 100644 --- a/src/OpenIPC.Viewer.Core/Services/CameraDirectoryService.cs +++ b/src/OpenIPC.Viewer.Core/Services/CameraDirectoryService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; using OpenIPC.Viewer.Core.Persistence; using OpenIPC.Viewer.Core.Platform; @@ -79,7 +80,8 @@ public async Task AddAsync(NewCameraRequest req, CancellationToken ct) CreatedAt: now, UpdatedAt: now, StreamQualityOverride: req.StreamQualityOverride, - SshPort: req.SshPort); + SshPort: req.SshPort, + Analytics: req.Analytics); return await _cameras.AddAsync(camera, ct).ConfigureAwait(false); } @@ -110,12 +112,26 @@ public async Task UpdateAsync(CameraId id, UpdateCameraRequest req, Cancellation GroupId = req.GroupId, StreamQualityOverride = req.StreamQualityOverride, SshPort = req.SshPort, + // Null keeps the stored analytics config (mirrors credentials). + Analytics = req.Analytics ?? existing.Analytics, UpdatedAt = DateTime.UtcNow, }; await _cameras.UpdateAsync(updated, ct).ConfigureAwait(false); } + /// + /// Updates only a camera's analytics config (Phase 15) — used by the AI + /// control center / tile toggles without touching credentials or geometry. + /// + public async Task SetAnalyticsAsync(CameraId id, AnalyticsSettings analytics, CancellationToken ct) + { + var existing = await _cameras.GetAsync(id, ct).ConfigureAwait(false) + ?? throw new InvalidOperationException($"Camera {id} not found"); + var updated = existing with { Analytics = analytics, UpdatedAt = DateTime.UtcNow }; + await _cameras.UpdateAsync(updated, ct).ConfigureAwait(false); + } + public Task UpdateSortOrdersAsync(IReadOnlyDictionary orders, CancellationToken ct) => _cameras.UpdateSortOrdersAsync(orders, ct); diff --git a/src/OpenIPC.Viewer.Core/Settings/IUserSettingsAccessor.cs b/src/OpenIPC.Viewer.Core/Settings/IUserSettingsAccessor.cs index 243278c..d4e24ad 100644 --- a/src/OpenIPC.Viewer.Core/Settings/IUserSettingsAccessor.cs +++ b/src/OpenIPC.Viewer.Core/Settings/IUserSettingsAccessor.cs @@ -1,3 +1,5 @@ +using OpenIPC.Viewer.Core.Analytics; + namespace OpenIPC.Viewer.Core.Settings; // Thin read-only view onto the UI's UserSettings that Core services can @@ -22,4 +24,7 @@ public interface IUserSettingsAccessor bool SshStrictHostKey { get; } int SshDefaultPort { get; } string MajesticConfigPath { get; } + + // Local AI analytics acceleration preference (Phase 15.2). + AiAcceleration AiAcceleration { get; } } diff --git a/src/OpenIPC.Viewer.Infrastructure/Persistence/Migrations/010_ai_analytics.sql b/src/OpenIPC.Viewer.Infrastructure/Persistence/Migrations/010_ai_analytics.sql new file mode 100644 index 0000000..300186b --- /dev/null +++ b/src/OpenIPC.Viewer.Infrastructure/Persistence/Migrations/010_ai_analytics.sql @@ -0,0 +1,9 @@ +-- Phase 15 — per-camera AI object detection. Disabled by default. AiClasses is +-- a CSV of COCO class ids to keep (NULL/empty = all classes). Existing rows +-- inherit the defaults, preserving current behavior. +ALTER TABLE Cameras ADD COLUMN AiEnabled INTEGER NOT NULL DEFAULT 0; +ALTER TABLE Cameras ADD COLUMN AiClasses TEXT; +ALTER TABLE Cameras ADD COLUMN AiThreshold REAL NOT NULL DEFAULT 0.5; +ALTER TABLE Cameras ADD COLUMN AiFps INTEGER NOT NULL DEFAULT 3; +ALTER TABLE Cameras ADD COLUMN AiAutoRecord INTEGER NOT NULL DEFAULT 0; +ALTER TABLE Cameras ADD COLUMN AiPostEventSeconds INTEGER NOT NULL DEFAULT 15; diff --git a/src/OpenIPC.Viewer.Infrastructure/Persistence/SqliteCameraRepository.cs b/src/OpenIPC.Viewer.Infrastructure/Persistence/SqliteCameraRepository.cs index 8b015f4..16576d0 100644 --- a/src/OpenIPC.Viewer.Infrastructure/Persistence/SqliteCameraRepository.cs +++ b/src/OpenIPC.Viewer.Infrastructure/Persistence/SqliteCameraRepository.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using Dapper; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; using OpenIPC.Viewer.Core.Persistence; using OpenIPC.Viewer.Core.Video; @@ -47,13 +48,15 @@ INSERT INTO Cameras ( RtspMainUri, RtspSubUri, UsernameRef, PasswordRef, OnvifEnabled, OnvifProfileToken, ChipModel, FirmwareVersion, IncludedInGrid, HasPtz, IsMajestic, SortOrder, CreatedAt, UpdatedAt, - StreamQualityOverride, SshPort) + StreamQualityOverride, SshPort, + AiEnabled, AiClasses, AiThreshold, AiFps, AiAutoRecord, AiPostEventSeconds) VALUES ( @Id, @GroupId, @Name, @Host, @OnvifPort, @HttpPort, @RtspMainUri, @RtspSubUri, @UsernameRef, @PasswordRef, @OnvifEnabled, @OnvifProfileToken, @ChipModel, @FirmwareVersion, @IncludedInGrid, @HasPtz, @IsMajestic, @SortOrder, @CreatedAt, @UpdatedAt, - @StreamQualityOverride, @SshPort); + @StreamQualityOverride, @SshPort, + @AiEnabled, @AiClasses, @AiThreshold, @AiFps, @AiAutoRecord, @AiPostEventSeconds); """, ToRow(camera), transaction: tx).ConfigureAwait(false); await tx.CommitAsync(ct).ConfigureAwait(false); @@ -86,7 +89,13 @@ UPDATE Cameras SET SortOrder = @SortOrder, UpdatedAt = @UpdatedAt, StreamQualityOverride = @StreamQualityOverride, - SshPort = @SshPort + SshPort = @SshPort, + AiEnabled = @AiEnabled, + AiClasses = @AiClasses, + AiThreshold = @AiThreshold, + AiFps = @AiFps, + AiAutoRecord = @AiAutoRecord, + AiPostEventSeconds = @AiPostEventSeconds WHERE Id = @Id; """, ToRow(camera), transaction: tx).ConfigureAwait(false); @@ -144,7 +153,14 @@ await conn.ExecuteAsync( CreatedAt: DateTime.Parse(row.CreatedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), UpdatedAt: DateTime.Parse(row.UpdatedAt, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), StreamQualityOverride: (StreamQualityOverride)row.StreamQualityOverride, - SshPort: row.SshPort is null ? null : (int)row.SshPort.Value); + SshPort: row.SshPort is null ? null : (int)row.SshPort.Value, + Analytics: new AnalyticsSettings( + Enabled: row.AiEnabled != 0, + ClassIds: ParseClassIds(row.AiClasses), + ConfidenceThreshold: (float)row.AiThreshold, + AnalyticsFps: row.AiFps, + AutoRecord: row.AiAutoRecord != 0, + PostEventSeconds: row.AiPostEventSeconds)); private static object ToRow(Camera c) => new { @@ -170,8 +186,30 @@ await conn.ExecuteAsync( UpdatedAt = c.UpdatedAt.ToUniversalTime().ToString("o", CultureInfo.InvariantCulture), StreamQualityOverride = (int)c.StreamQualityOverride, c.SshPort, + AiEnabled = c.AnalyticsOrDefault.Enabled ? 1 : 0, + AiClasses = FormatClassIds(c.AnalyticsOrDefault.ClassIds), + AiThreshold = c.AnalyticsOrDefault.ConfidenceThreshold, + AiFps = c.AnalyticsOrDefault.AnalyticsFps, + AiAutoRecord = c.AnalyticsOrDefault.AutoRecord ? 1 : 0, + AiPostEventSeconds = c.AnalyticsOrDefault.PostEventSeconds, }; + // AiClasses is a CSV of COCO class ids; null/empty means "all classes". + private static IReadOnlyCollection? ParseClassIds(string? csv) + { + if (string.IsNullOrWhiteSpace(csv)) return null; + var ids = new List(); + foreach (var part in csv.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (int.TryParse(part, NumberStyles.Integer, CultureInfo.InvariantCulture, out var id)) + ids.Add(id); + } + return ids.Count == 0 ? null : ids; + } + + private static string? FormatClassIds(IReadOnlyCollection? ids) => + ids is { Count: > 0 } ? string.Join(',', ids) : null; + private sealed class CameraRow { public string Id { get; init; } = default!; @@ -196,5 +234,11 @@ private sealed class CameraRow public string UpdatedAt { get; init; } = default!; public int StreamQualityOverride { get; init; } public long? SshPort { get; init; } + public int AiEnabled { get; init; } + public string? AiClasses { get; init; } + public double AiThreshold { get; init; } + public int AiFps { get; init; } + public int AiAutoRecord { get; init; } + public int AiPostEventSeconds { get; init; } } } From b37797d9718c067e6eb04ceb425ad924f05dcdb3 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 22:03:39 +0300 Subject: [PATCH 04/13] feat(analytics): auto-record on detection + smart auto-stop (15.6) - AutoRecordWindow: pure start/extend/stop state machine with cooldown - AutoRecordCoordinator: detections -> RecordingService start, 1Hz quiet-stop tick - Only stops recordings it started; manual recordings are never auto-stopped - Graceful stop via RecordingService (avoids corrupt MP4); Failed event for logging - AutoRecordWindow unit tests (extend, quiet-stop, cooldown) --- .../SharedComposition.cs | 3 + .../Analytics/AutoRecordCoordinator.cs | 113 ++++++++++++++++++ .../Analytics/AutoRecordWindow.cs | 57 +++++++++ .../Analytics/AutoRecordWindowTests.cs | 59 +++++++++ 4 files changed, 232 insertions(+) create mode 100644 src/OpenIPC.Viewer.Core/Analytics/AutoRecordCoordinator.cs create mode 100644 src/OpenIPC.Viewer.Core/Analytics/AutoRecordWindow.cs create mode 100644 tests/OpenIPC.Viewer.Core.Tests/Analytics/AutoRecordWindowTests.cs diff --git a/src/OpenIPC.Viewer.Composition/SharedComposition.cs b/src/OpenIPC.Viewer.Composition/SharedComposition.cs index 7aa4965..64d679d 100644 --- a/src/OpenIPC.Viewer.Composition/SharedComposition.cs +++ b/src/OpenIPC.Viewer.Composition/SharedComposition.cs @@ -83,6 +83,9 @@ public static IServiceCollection AddSharedServices(this IServiceCollection servi OpenIPC.Viewer.Analytics.OnnxObjectDetector>(); services.AddSingleton(); + // Auto-record on detection (Phase 15.6). Started by the App analytics + // bootstrap once the engine is initialized. + services.AddSingleton(); // Recording lifecycle (IRecorder itself is registered by the platform // host — FFmpeg subprocess on desktop, FFmpegKit on Android, etc). diff --git a/src/OpenIPC.Viewer.Core/Analytics/AutoRecordCoordinator.cs b/src/OpenIPC.Viewer.Core/Analytics/AutoRecordCoordinator.cs new file mode 100644 index 0000000..e22d797 --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/AutoRecordCoordinator.cs @@ -0,0 +1,113 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Persistence; +using OpenIPC.Viewer.Core.Recording; + +namespace OpenIPC.Viewer.Core.Analytics; + +// Drives auto-record from detections (Phase 15.6). Subscribes to the engine's +// results; when a camera with AutoRecord on detects something it starts +// recording and arms a smart-stop window. A 1 Hz tick stops recordings whose +// window has gone quiet. Only stops recordings IT started — a manual recording +// is never auto-stopped. Stop goes through RecordingService for a graceful MP4 +// close (the dashboard's corrupted-file trap). +public sealed class AutoRecordCoordinator : IDisposable +{ + private const int CooldownSeconds = 10; + + private readonly IAnalyticsEngine _engine; + private readonly RecordingService _recording; + private readonly ICameraRepository _cameras; + + private readonly ConcurrentDictionary _windows = new(); + private readonly HashSet _autoStarted = new(); + private readonly object _startedGate = new(); + + private IDisposable? _subscription; + private Timer? _tick; + + // Surfaces start/stop failures to the App layer for logging — Core stays + // package-dep free (same pattern as RecordingService.BookkeepingFailed). + public event EventHandler? Failed; + + public AutoRecordCoordinator(IAnalyticsEngine engine, RecordingService recording, + ICameraRepository cameras) + { + _engine = engine; + _recording = recording; + _cameras = cameras; + } + + public void Start() + { + _subscription ??= _engine.Results.Subscribe(new ResultObserver(r => _ = OnResultAsync(r))); + _tick ??= new Timer(_ => _ = TickAsync(), null, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1)); + } + + private async Task OnResultAsync(DetectionResult result) + { + try + { + if (result.Detections.Count == 0) return; + + // Live read so toggling AutoRecord off takes effect immediately. + var camera = await _cameras.GetAsync(result.CameraId, CancellationToken.None).ConfigureAwait(false); + var settings = camera?.AnalyticsOrDefault; + if (settings is null || !settings.AutoRecord) return; + + var window = _windows.GetOrAdd(result.CameraId, + _ => new AutoRecordWindow(settings.PostEventSeconds, CooldownSeconds)); + + if (window.OnDetection(result.OccurredAt) != AutoRecordAction.Start) return; + if (_recording.IsRecording(result.CameraId)) return; // manual recording already running + + await _recording.StartAsync(result.CameraId, CancellationToken.None).ConfigureAwait(false); + lock (_startedGate) _autoStarted.Add(result.CameraId); + } + catch (Exception ex) + { + Failed?.Invoke(this, ex); + } + } + + private async Task TickAsync() + { + var now = DateTime.UtcNow; + foreach (var kv in _windows) + { + if (kv.Value.OnTick(now) != AutoRecordAction.Stop) continue; + + bool ours; + lock (_startedGate) ours = _autoStarted.Remove(kv.Key); + if (!ours || !_recording.IsRecording(kv.Key)) continue; + + try + { + await _recording.StopAsync(kv.Key, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception ex) + { + Failed?.Invoke(this, ex); + } + } + } + + public void Dispose() + { + _subscription?.Dispose(); + _tick?.Dispose(); + } + + private sealed class ResultObserver : IObserver + { + private readonly Action _onNext; + public ResultObserver(Action onNext) => _onNext = onNext; + public void OnNext(DetectionResult value) => _onNext(value); + public void OnError(Exception error) { } + public void OnCompleted() { } + } +} diff --git a/src/OpenIPC.Viewer.Core/Analytics/AutoRecordWindow.cs b/src/OpenIPC.Viewer.Core/Analytics/AutoRecordWindow.cs new file mode 100644 index 0000000..e44141e --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Analytics/AutoRecordWindow.cs @@ -0,0 +1,57 @@ +using System; + +namespace OpenIPC.Viewer.Core.Analytics; + +public enum AutoRecordAction +{ + None = 0, + Start, + Stop, +} + +// Smart auto-stop state machine (Phase 15.6). Recording starts on the first +// detection and keeps running until postEventSeconds elapse with no new +// detection; each detection extends the window. A cooldown after a stop avoids +// start/stop flapping on a lone stray detection. Pure (clock passed in) so the +// extend/stop/cooldown behaviour unit-tests deterministically. +public sealed class AutoRecordWindow +{ + private readonly TimeSpan _postEvent; + private readonly TimeSpan _cooldown; + private DateTime _lastDetection = DateTime.MinValue; + private DateTime _stoppedAt = DateTime.MinValue; + private bool _active; + + public AutoRecordWindow(int postEventSeconds, int cooldownSeconds = 0) + { + if (postEventSeconds < 1) postEventSeconds = 1; + _postEvent = TimeSpan.FromSeconds(postEventSeconds); + _cooldown = TimeSpan.FromSeconds(Math.Max(0, cooldownSeconds)); + } + + public bool IsActive => _active; + + public AutoRecordAction OnDetection(DateTime now) + { + // During the post-stop cooldown, ignore detections so a single stray + // hit can't immediately restart recording. + if (!_active && _stoppedAt != DateTime.MinValue && now - _stoppedAt < _cooldown) + return AutoRecordAction.None; + + _lastDetection = now; + if (_active) return AutoRecordAction.None; + _active = true; + return AutoRecordAction.Start; + } + + public AutoRecordAction OnTick(DateTime now) + { + if (_active && now - _lastDetection >= _postEvent) + { + _active = false; + _stoppedAt = now; + return AutoRecordAction.Stop; + } + return AutoRecordAction.None; + } +} diff --git a/tests/OpenIPC.Viewer.Core.Tests/Analytics/AutoRecordWindowTests.cs b/tests/OpenIPC.Viewer.Core.Tests/Analytics/AutoRecordWindowTests.cs new file mode 100644 index 0000000..b227c09 --- /dev/null +++ b/tests/OpenIPC.Viewer.Core.Tests/Analytics/AutoRecordWindowTests.cs @@ -0,0 +1,59 @@ +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Core.Tests.Analytics; + +// Smart auto-stop window (Phase 15.6): start on first detection, extend on each +// new one, stop after a quiet period, cooldown before restarting. +public sealed class AutoRecordWindowTests +{ + private static readonly DateTime T0 = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + [Fact] + public void FirstDetection_StartsRecording() + { + var w = new AutoRecordWindow(postEventSeconds: 15); + Assert.Equal(AutoRecordAction.Start, w.OnDetection(T0)); + Assert.True(w.IsActive); + } + + [Fact] + public void DetectionWhileActive_DoesNotStartAgain() + { + var w = new AutoRecordWindow(15); + w.OnDetection(T0); + Assert.Equal(AutoRecordAction.None, w.OnDetection(T0.AddSeconds(5))); + } + + [Fact] + public void Stops_AfterQuietPeriod() + { + var w = new AutoRecordWindow(15); + w.OnDetection(T0); + Assert.Equal(AutoRecordAction.None, w.OnTick(T0.AddSeconds(10))); + Assert.Equal(AutoRecordAction.Stop, w.OnTick(T0.AddSeconds(15))); + Assert.False(w.IsActive); + } + + [Fact] + public void NewDetection_ExtendsTheWindow() + { + var w = new AutoRecordWindow(15); + w.OnDetection(T0); + w.OnDetection(T0.AddSeconds(10)); // extend: last = +10s + Assert.Equal(AutoRecordAction.None, w.OnTick(T0.AddSeconds(20))); // 20-10 = 10 < 15 + Assert.Equal(AutoRecordAction.Stop, w.OnTick(T0.AddSeconds(26))); // 26-10 = 16 >= 15 + } + + [Fact] + public void Cooldown_BlocksImmediateRestart_ThenAllows() + { + var w = new AutoRecordWindow(postEventSeconds: 5, cooldownSeconds: 10); + w.OnDetection(T0); + w.OnTick(T0.AddSeconds(5)); // Stop at +5s + + // Within cooldown (10s after stop) → ignored. + Assert.Equal(AutoRecordAction.None, w.OnDetection(T0.AddSeconds(10))); + // After cooldown → starts again. + Assert.Equal(AutoRecordAction.Start, w.OnDetection(T0.AddSeconds(16))); + } +} From 3135c54f6b380fdef53d4c533b8c556eea4cffba Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 22:09:36 +0300 Subject: [PATCH 05/13] feat(analytics): detection events via the ingestion path (15.7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New EventKind.Detection; MotionTick carries optional Kind + Summary - EventIngestionService threads kind/summary into opened + finalized events - AnalyticsMotionEventSource bridges engine results -> motion ticks (debounce, quiet-close, live observable reused) - Detection summary "person ×2, car ×1"; summary unit tests --- .../SharedComposition.cs | 2 + .../Events/AnalyticsMotionEventSource.cs | 66 +++++++++++++++++++ src/OpenIPC.Viewer.Core/Events/CameraEvent.cs | 1 + .../Events/EventIngestionService.cs | 17 +++-- .../Events/IMotionEventSource.cs | 10 ++- .../Analytics/DetectionSummaryTests.cs | 23 +++++++ 6 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 src/OpenIPC.Viewer.Core/Events/AnalyticsMotionEventSource.cs create mode 100644 tests/OpenIPC.Viewer.Core.Tests/Analytics/DetectionSummaryTests.cs diff --git a/src/OpenIPC.Viewer.Composition/SharedComposition.cs b/src/OpenIPC.Viewer.Composition/SharedComposition.cs index 64d679d..922e7ac 100644 --- a/src/OpenIPC.Viewer.Composition/SharedComposition.cs +++ b/src/OpenIPC.Viewer.Composition/SharedComposition.cs @@ -94,6 +94,8 @@ public static IServiceCollection AddSharedServices(this IServiceCollection servi // Events services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); + // AI detections feed the same ingestion path as motion (Phase 15.7). + services.AddSingleton(); services.AddSingleton(); // UI services diff --git a/src/OpenIPC.Viewer.Core/Events/AnalyticsMotionEventSource.cs b/src/OpenIPC.Viewer.Core/Events/AnalyticsMotionEventSource.cs new file mode 100644 index 0000000..340e2fd --- /dev/null +++ b/src/OpenIPC.Viewer.Core/Events/AnalyticsMotionEventSource.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Entities; + +namespace OpenIPC.Viewer.Core.Events; + +// Turns AI detections into motion ticks (Phase 15.7) so detection events flow +// through the same ingestion path as motion — debounce, quiet-close, and the +// live Events observable all come for free. Each tick carries Kind=Detection +// and a class-count summary like "person ×2, car ×1". +public sealed class AnalyticsMotionEventSource : IMotionEventSource +{ + private readonly IAnalyticsEngine _engine; + + public AnalyticsMotionEventSource(IAnalyticsEngine engine) => _engine = engine; + + public string Name => "analytics"; + + public IDisposable Watch(CameraId cameraId, IObserver observer, CancellationToken ct) + => _engine.Results.Subscribe(new ResultObserver(cameraId, observer)); + + // Builds "person ×2, car ×1" ordered by descending count. + public static string Summarize(IReadOnlyList detections) + { + var counts = new Dictionary(StringComparer.Ordinal); + foreach (var d in detections) + counts[d.ClassName] = counts.TryGetValue(d.ClassName, out var n) ? n + 1 : 1; + + var parts = new List>(counts); + parts.Sort((a, b) => b.Value.CompareTo(a.Value)); + + var sb = new StringBuilder(); + foreach (var kv in parts) + { + if (sb.Length > 0) sb.Append(", "); + sb.Append(kv.Key).Append(" ×").Append(kv.Value); + } + return sb.ToString(); + } + + private sealed class ResultObserver : IObserver + { + private readonly CameraId _cameraId; + private readonly IObserver _target; + + public ResultObserver(CameraId cameraId, IObserver target) + { + _cameraId = cameraId; + _target = target; + } + + public void OnNext(DetectionResult value) + { + if (value.CameraId != _cameraId || value.Detections.Count == 0) return; + _target.OnNext(new MotionTick( + _cameraId, value.OccurredAt, "analytics", + EventKind.Detection, Summarize(value.Detections))); + } + + public void OnError(Exception error) { } + public void OnCompleted() { } + } +} diff --git a/src/OpenIPC.Viewer.Core/Events/CameraEvent.cs b/src/OpenIPC.Viewer.Core/Events/CameraEvent.cs index 6ce197f..bd9fd4b 100644 --- a/src/OpenIPC.Viewer.Core/Events/CameraEvent.cs +++ b/src/OpenIPC.Viewer.Core/Events/CameraEvent.cs @@ -8,6 +8,7 @@ public enum EventKind Motion = 0, Connection, // RTSP connect / disconnect (future) Snapshot, // user-initiated snapshot (future) + Detection, // AI object detection (Phase 15) — Summary holds class counts } public enum EventSeverity diff --git a/src/OpenIPC.Viewer.Core/Events/EventIngestionService.cs b/src/OpenIPC.Viewer.Core/Events/EventIngestionService.cs index 806bf6e..e6950ae 100644 --- a/src/OpenIPC.Viewer.Core/Events/EventIngestionService.cs +++ b/src/OpenIPC.Viewer.Core/Events/EventIngestionService.cs @@ -68,11 +68,13 @@ private void OnTick(MotionTick tick) { if (!_open.TryGetValue(tick.CameraId, out state)) { - state = new OpenEventState(EventId.New(), tick.CameraId, tick.At, tick.Source); + state = new OpenEventState(EventId.New(), tick.CameraId, tick.At, tick.Source, tick.Kind); _open[tick.CameraId] = state; isNew = true; } state.LastTickAt = tick.At; + // Latest tick's summary wins (e.g. peak class counts during the episode). + if (tick.Summary is not null) state.Summary = tick.Summary; state.QuietTimer?.Change(CloseAfterQuiet, Timeout.InfiniteTimeSpan); state.QuietTimer ??= new Timer(CloseQuietState, state, CloseAfterQuiet, Timeout.InfiniteTimeSpan); } @@ -86,12 +88,12 @@ private async Task OpenAsync(OpenEventState state) var ev = new CameraEvent( Id: state.EventId, CameraId: state.CameraId, - Kind: EventKind.Motion, + Kind: state.Kind, Severity: EventSeverity.Info, OccurredAt: state.StartedAt, EndedAt: null, Source: state.Source, - Summary: null); + Summary: state.Summary); try { await _repo.AddAsync(ev, CancellationToken.None).ConfigureAwait(false); } catch { /* surface via finalize event instead — swallow here */ } @@ -122,12 +124,12 @@ private async Task FinalizeAsync(OpenEventState state) var finalized = new CameraEvent( Id: state.EventId, CameraId: state.CameraId, - Kind: EventKind.Motion, + Kind: state.Kind, Severity: EventSeverity.Info, OccurredAt: state.StartedAt, EndedAt: state.LastTickAt, Source: state.Source, - Summary: null); + Summary: state.Summary); try { await _repo.UpdateAsync(finalized, CancellationToken.None).ConfigureAwait(false); } catch { /* swallow */ } @@ -169,15 +171,18 @@ private sealed class OpenEventState public DateTime StartedAt { get; } public DateTime LastTickAt { get; set; } public string? Source { get; } + public EventKind Kind { get; } + public string? Summary { get; set; } public Timer? QuietTimer { get; set; } - public OpenEventState(EventId id, CameraId cam, DateTime started, string? source) + public OpenEventState(EventId id, CameraId cam, DateTime started, string? source, EventKind kind) { EventId = id; CameraId = cam; StartedAt = started; LastTickAt = started; Source = source; + Kind = kind; } } diff --git a/src/OpenIPC.Viewer.Core/Events/IMotionEventSource.cs b/src/OpenIPC.Viewer.Core/Events/IMotionEventSource.cs index 25085ab..0c3f7aa 100644 --- a/src/OpenIPC.Viewer.Core/Events/IMotionEventSource.cs +++ b/src/OpenIPC.Viewer.Core/Events/IMotionEventSource.cs @@ -20,4 +20,12 @@ public interface IMotionEventSource IDisposable Watch(CameraId cameraId, IObserver observer, CancellationToken ct); } -public readonly record struct MotionTick(CameraId CameraId, DateTime At, string Source); +// Kind/Summary default to a plain motion tick; analytics sources set Kind to +// Detection and Summary to the class counts (Phase 15.7) so the same ingestion +// path produces both motion and detection events. +public readonly record struct MotionTick( + CameraId CameraId, + DateTime At, + string Source, + EventKind Kind = EventKind.Motion, + string? Summary = null); diff --git a/tests/OpenIPC.Viewer.Core.Tests/Analytics/DetectionSummaryTests.cs b/tests/OpenIPC.Viewer.Core.Tests/Analytics/DetectionSummaryTests.cs new file mode 100644 index 0000000..30a094c --- /dev/null +++ b/tests/OpenIPC.Viewer.Core.Tests/Analytics/DetectionSummaryTests.cs @@ -0,0 +1,23 @@ +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Events; + +namespace OpenIPC.Viewer.Core.Tests.Analytics; + +// Class-count summary that becomes the Detection event's Summary text (15.7). +public sealed class DetectionSummaryTests +{ + private static Detection Det(string cls) => new(0, cls, 0.9f, 0f, 0f, 0.1f, 0.1f); + + [Fact] + public void Summarize_CountsPerClass_OrderedByDescendingCount() + { + var detections = new[] { Det("car"), Det("person"), Det("person") }; + Assert.Equal("person ×2, car ×1", AnalyticsMotionEventSource.Summarize(detections)); + } + + [Fact] + public void Summarize_SingleClass() + { + Assert.Equal("person ×1", AnalyticsMotionEventSource.Summarize(new[] { Det("person") })); + } +} From e04c5f76bb8bdc0111afa676e3bc2b84aaafdf68 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 22:16:58 +0300 Subject: [PATCH 06/13] feat(analytics): tile detection overlay + engine bootstrap (15.5) - DetectionOverlay control: scales normalized boxes to tile, per-class colors + labels - CameraTileViewModel attaches/detaches frames per session; exposes Detections + counter - Analytics pauses with Smart Pause (suspended/non-playing tile -> isActive false) - AnalyticsBootstrap: lazy engine init + auto-record start on first enabled tile (best-effort) - GridPage: overlay above video + bottom-center object counter --- .../Controls/DetectionOverlay.cs | 81 +++++++++++++++++++ .../Services/AnalyticsBootstrap.cs | 60 ++++++++++++++ .../ViewModels/CameraTileViewModel.cs | 76 ++++++++++++++++- .../ViewModels/GridPageViewModel.cs | 10 ++- .../Views/Pages/GridPage.axaml | 12 +++ .../SharedComposition.cs | 2 + 6 files changed, 236 insertions(+), 5 deletions(-) create mode 100644 src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs create mode 100644 src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs diff --git a/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs b/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs new file mode 100644 index 0000000..e2d67a6 --- /dev/null +++ b/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs @@ -0,0 +1,81 @@ +using System.Collections.Generic; +using System.Globalization; +using Avalonia; +using Avalonia.Controls; +using Avalonia.Media; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.App.Controls; + +// Draws detection boxes + labels over the video tile (Phase 15.5). Boxes are +// normalized 0..1 in the source frame, so we just scale them to Bounds — no +// dependency on the model input size. Rendered directly (a Canvas of one box +// per visual would churn the tree); we repaint on each new detection result. +public sealed class DetectionOverlay : Control +{ + public static readonly StyledProperty?> DetectionsProperty = + AvaloniaProperty.Register?>(nameof(Detections)); + + public static readonly StyledProperty ShowBoxesProperty = + AvaloniaProperty.Register(nameof(ShowBoxes), defaultValue: true); + + // Distinct hues per class id so different objects read apart at a glance. + private static readonly Color[] Palette = + { + Color.FromRgb(0x4C, 0xAF, 0x50), Color.FromRgb(0x21, 0x96, 0xF3), + Color.FromRgb(0xFF, 0x98, 0x00), Color.FromRgb(0xE0, 0x40, 0x40), + Color.FromRgb(0x9C, 0x27, 0xB0), Color.FromRgb(0x00, 0xBC, 0xD4), + Color.FromRgb(0xFF, 0xEB, 0x3B), Color.FromRgb(0xFF, 0x40, 0x81), + }; + + static DetectionOverlay() + { + AffectsRender(DetectionsProperty, ShowBoxesProperty); + } + + public IReadOnlyList? Detections + { + get => GetValue(DetectionsProperty); + set => SetValue(DetectionsProperty, value); + } + + public bool ShowBoxes + { + get => GetValue(ShowBoxesProperty); + set => SetValue(ShowBoxesProperty, value); + } + + public override void Render(DrawingContext context) + { + base.Render(context); + var detections = Detections; + if (!ShowBoxes || detections is null || detections.Count == 0) return; + + var w = Bounds.Width; + var h = Bounds.Height; + if (w <= 0 || h <= 0) return; + + foreach (var d in detections) + { + var color = Palette[((d.ClassId % Palette.Length) + Palette.Length) % Palette.Length]; + var pen = new Pen(new SolidColorBrush(color), 2); + + var x = d.X * w; + var y = d.Y * h; + var bw = d.Width * w; + var bh = d.Height * h; + var box = new Rect(x, y, bw, bh); + context.DrawRectangle(null, pen, box); + + var label = $"{d.ClassName} {d.Confidence:0.00}"; + var text = new FormattedText(label, CultureInfo.InvariantCulture, FlowDirection.LeftToRight, + Typeface.Default, 11, Brushes.White); + + var labelW = text.Width + 8; + var labelH = text.Height + 2; + var labelY = y - labelH >= 0 ? y - labelH : y; // flip below the top edge if clipped + context.DrawRectangle(new SolidColorBrush(color), null, new Rect(x, labelY, labelW, labelH)); + context.DrawText(text, new Point(x + 4, labelY + 1)); + } + } +} diff --git a/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs b/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs new file mode 100644 index 0000000..5dba893 --- /dev/null +++ b/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.App.Services; + +// Lazily brings the analytics engine online (Phase 15). The first tile with +// analytics enabled calls EnsureStartedAsync, which downloads/loads the model, +// picks the execution provider, and starts the auto-record coordinator. Best- +// effort: a failure (e.g. offline first run, no model) is logged and analytics +// simply stays off — it never blocks the UI or crashes the app. +public sealed class AnalyticsBootstrap +{ + private readonly IAnalyticsEngine _engine; + private readonly AutoRecordCoordinator _autoRecord; + private readonly UserSettingsService _settings; + private readonly ILogger _log; + private readonly SemaphoreSlim _gate = new(1, 1); + private bool _started; + + public AnalyticsBootstrap( + IAnalyticsEngine engine, + AutoRecordCoordinator autoRecord, + UserSettingsService settings, + ILogger log) + { + _engine = engine; + _autoRecord = autoRecord; + _settings = settings; + _log = log; + _autoRecord.Failed += (_, ex) => _log.LogWarning(ex, "Auto-record error."); + } + + public bool IsReady => _engine.IsReady; + + public async Task EnsureStartedAsync() + { + if (_started) return; + await _gate.WaitAsync().ConfigureAwait(false); + try + { + if (_started) return; + await _engine.InitializeAsync(_settings.AiAcceleration, CancellationToken.None).ConfigureAwait(false); + _autoRecord.Start(); + _started = true; + _log.LogInformation("Analytics engine started on {Provider}.", _engine.ActiveProvider); + } + catch (Exception ex) + { + // Leave _started false so a later enable retries (e.g. once online). + _log.LogError(ex, "Analytics engine failed to start; analytics stays off."); + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs index 5d9e01d..82068ee 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; @@ -9,7 +10,9 @@ using Microsoft.Extensions.Logging; using OpenIPC.Viewer.App.Messages; using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Events; using OpenIPC.Viewer.Core.Services; using OpenIPC.Viewer.Core.Snapshots; using OpenIPC.Viewer.Core.Video; @@ -22,6 +25,8 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl private readonly CameraDirectoryService _directory; private readonly UserSettingsService _userSettings; private readonly ISnapshotService _snapshots; + private readonly IAnalyticsEngine _analytics; + private readonly AnalyticsBootstrap _analyticsBootstrap; private readonly ILogger _logger; // Auto SD/HD (Phase 12.2): substream in the grid, mainstream when a single @@ -31,8 +36,10 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl private StreamQuality _quality = StreamQuality.Sub; private IDisposable? _stateSub; private IDisposable? _telemetrySub; + private IDisposable? _resultsSub; private bool _started; private bool _disposed; + private bool _suspended; public Camera Camera { get; } @@ -54,6 +61,20 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl [NotifyPropertyChangedFor(nameof(ErrorDetail))] private string? _errorMessage; + // Latest detections for this tile (Phase 15.5), normalized 0..1 boxes drawn + // by the DetectionOverlay control. Updated off the inference worker thread. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(DetectionCounter))] + [NotifyPropertyChangedFor(nameof(HasDetections))] + private IReadOnlyList _detections = Array.Empty(); + + public bool AnalyticsEnabled => Camera.AnalyticsOrDefault.Enabled; + + // Bottom-center counter badge, e.g. "person ×2, car ×1". + public string DetectionCounter => AnalyticsMotionEventSource.Summarize(Detections); + + public bool HasDetections => AnalyticsEnabled && Detections.Count > 0; + public string Name => Camera.Name; public string StateLabel => State switch { @@ -116,6 +137,8 @@ public CameraTileViewModel( CameraDirectoryService directory, UserSettingsService userSettings, ISnapshotService snapshots, + IAnalyticsEngine analytics, + AnalyticsBootstrap analyticsBootstrap, ILogger logger) { Camera = camera; @@ -123,9 +146,17 @@ public CameraTileViewModel( _directory = directory; _userSettings = userSettings; _snapshots = snapshots; + _analytics = analytics; + _analyticsBootstrap = analyticsBootstrap; _logger = logger; _coordinator.Invalidated += OnCoordinatorInvalidated; + + // Always listen for results for this camera; they only arrive while the + // tile is attached + the engine is ready, so this is cheap otherwise. + _resultsSub = _analytics.Results + .Where(r => r.CameraId == Camera.Id) + .Subscribe(r => Dispatcher.UIThread.Post(() => Detections = r.Detections)); } // Set the stream quality before the first ActivateAsync. No-op once started @@ -145,6 +176,7 @@ public async Task SetQualityAsync(StreamQuality quality, CancellationToken ct) if (_disposed || quality == _quality) return; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; @@ -188,6 +220,8 @@ public async Task ActivateAsync(CancellationToken ct) if (session.State == SessionState.Idle) await session.StartAsync(ct).ConfigureAwait(true); + + AttachAnalytics(session); } catch (Exception ex) { @@ -196,6 +230,27 @@ public async Task ActivateAsync(CancellationToken ct) } } + // Tap this session's frames for object detection (Phase 15.3) when the + // camera has analytics on. Kicks the engine bootstrap (model load) in the + // background; frames are dropped until it's ready. isActive gates analytics + // off for a Smart-Paused / non-playing tile. + private void AttachAnalytics(IVideoSession session) + { + if (!Camera.AnalyticsOrDefault.Enabled) return; + _ = _analyticsBootstrap.EnsureStartedAsync(); + _analytics.Attach( + Camera.Id, + session.Frames, + () => Camera.AnalyticsOrDefault, + () => !_suspended && State == SessionState.Playing); + } + + private void DetachAnalytics() + { + _analytics.Detach(Camera.Id); + Detections = Array.Empty(); + } + // Coordinator dropped every cached session (e.g. RtspTransport flipped in // Settings). Our current Session ref is now disposed — drop subscriptions, // reset state, and re-Acquire on the UI thread so observable updates land @@ -209,6 +264,7 @@ private void OnCoordinatorInvalidated(object? sender, EventArgs e) { _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); Session = null; State = SessionState.Idle; _started = false; @@ -261,6 +317,7 @@ private async Task RetryAsync() ErrorMessage = null; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; @@ -278,9 +335,20 @@ private void Close() => WeakReferenceMessenger.Default.Send(new CloseTileMessage(Camera.Id)); // Smart Pause (Phase 12.1): suspend/resume decode without dropping the - // session, so the last frame stays frozen for an instant resume. - public void Pause() => Session?.PauseDecode(); - public void Resume() => Session?.Resume(); + // session, so the last frame stays frozen for an instant resume. Analytics + // pauses with the tile (12.1 ↔ 15.3) — isActive reads _suspended. + public void Pause() + { + _suspended = true; + Session?.PauseDecode(); + Detections = Array.Empty(); + } + + public void Resume() + { + _suspended = false; + Session?.Resume(); + } public async ValueTask DisposeAsync() { @@ -288,6 +356,8 @@ public async ValueTask DisposeAsync() _coordinator.Invalidated -= OnCoordinatorInvalidated; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + _resultsSub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; diff --git a/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs index b3fcfd2..77c01d0 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs @@ -27,6 +27,8 @@ public sealed partial class GridPageViewModel : ViewModelBase, private readonly LiveStreamCoordinator _coordinator; private readonly UserSettingsService _userSettings; private readonly ISnapshotService _snapshots; + private readonly OpenIPC.Viewer.Core.Analytics.IAnalyticsEngine _analytics; + private readonly AnalyticsBootstrap _analyticsBootstrap; private readonly ILoggerFactory _loggerFactory; private readonly ILogger _logger; @@ -52,12 +54,16 @@ public GridPageViewModel( LiveStreamCoordinator coordinator, UserSettingsService userSettings, ISnapshotService snapshots, + OpenIPC.Viewer.Core.Analytics.IAnalyticsEngine analytics, + AnalyticsBootstrap analyticsBootstrap, ILoggerFactory loggerFactory) { _directory = directory; _coordinator = coordinator; _userSettings = userSettings; _snapshots = snapshots; + _analytics = analytics; + _analyticsBootstrap = analyticsBootstrap; _loggerFactory = loggerFactory; _logger = loggerFactory.CreateLogger(); @@ -140,7 +146,7 @@ private async Task RefreshTilesAsync(CancellationToken ct) try { await existing.DisposeAsync().ConfigureAwait(true); } catch (Exception ex) { _logger.LogWarning(ex, "Error releasing stale tile for {Camera}", camera.Name); } - var rebuilt = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _loggerFactory.CreateLogger()); + var rebuilt = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _analytics, _analyticsBootstrap, _loggerFactory.CreateLogger()); rebuilt.SetInitialQuality(quality); Tiles.Insert(idx, rebuilt); try { await rebuilt.ActivateAsync(ct).ConfigureAwait(true); } @@ -148,7 +154,7 @@ private async Task RefreshTilesAsync(CancellationToken ct) continue; } - var tile = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _loggerFactory.CreateLogger()); + var tile = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _analytics, _analyticsBootstrap, _loggerFactory.CreateLogger()); tile.SetInitialQuality(quality); Tiles.Add(tile); try { await tile.ActivateAsync(ct).ConfigureAwait(true); } diff --git a/src/OpenIPC.Viewer.App/Views/Pages/GridPage.axaml b/src/OpenIPC.Viewer.App/Views/Pages/GridPage.axaml index 4ee99ef..58a4fb6 100644 --- a/src/OpenIPC.Viewer.App/Views/Pages/GridPage.axaml +++ b/src/OpenIPC.Viewer.App/Views/Pages/GridPage.axaml @@ -75,12 +75,24 @@ PointerPressed="OnTilePointerPressed"> + + + + + + + + (); + // Lazily initializes the engine on first analytics-enabled tile (15.4). + services.AddSingleton(); // Recording lifecycle (IRecorder itself is registered by the platform // host — FFmpeg subprocess on desktop, FFmpegKit on Android, etc). From 552f60fdf22bd9715b28569cbeeb392ae9cf98ac Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 22:28:32 +0300 Subject: [PATCH 07/13] feat(analytics): camera editor analytics tab + settings toggle (15.4 UI) - Camera editor: enable, class toggles (curated COCO subset), threshold, fps, auto-record + post-event - Hydrates/persists AnalyticsSettings through New/UpdateCameraRequest - Settings: "Force CPU for AI analytics" toggle wired to AiAcceleration - EN/RU localization for the new strings --- src/OpenIPC.Viewer.App/Services/Localizer.cs | 16 ++++ .../Dialogs/CameraEditorViewModel.cs | 73 ++++++++++++++++++- .../ViewModels/SettingsPageViewModel.cs | 6 ++ .../Views/Dialogs/CameraEditorContent.axaml | 41 +++++++++++ .../Views/Pages/SettingsPage.axaml | 4 + 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/src/OpenIPC.Viewer.App/Services/Localizer.cs b/src/OpenIPC.Viewer.App/Services/Localizer.cs index c265f6c..9e5f56e 100644 --- a/src/OpenIPC.Viewer.App/Services/Localizer.cs +++ b/src/OpenIPC.Viewer.App/Services/Localizer.cs @@ -153,6 +153,7 @@ private static LangCode DetectSystem() ["Settings.Video.TelemetryOverlay"] = "Show telemetry overlay on live view", ["Settings.Video.AutoSdHd"] = "Auto SD/HD (sub in grid, main when full-screen)", + ["Settings.Video.AiForceCpu"] = "Force CPU for AI analytics (no GPU acceleration)", ["Settings.Video.MaxGridSessions"] = "Max concurrent grid sessions", ["Settings.Video.RtspTransport"] = "Default RTSP transport", ["Settings.Video.NetworkInterface"] = "Network interface (discovery)", @@ -192,6 +193,13 @@ private static LangCode DetectSystem() ["CameraEditor.Quality.Auto"] = "Auto (SD/HD)", ["CameraEditor.Quality.AlwaysHd"] = "Always HD (main)", ["CameraEditor.Quality.AlwaysSd"] = "Always SD (sub)", + ["CameraEditor.Analytics.Section"] = "AI analytics", + ["CameraEditor.Analytics.Enable"] = "Enable object detection", + ["CameraEditor.Analytics.Classes"] = "Classes", + ["CameraEditor.Analytics.Threshold"] = "Confidence threshold", + ["CameraEditor.Analytics.Fps"] = "Analysis FPS", + ["CameraEditor.Analytics.AutoRecord"] = "Auto-record on detection", + ["CameraEditor.Analytics.PostEvent"] = "Keep recording after (s)", ["CameraEditor.Placeholder.Name"] = "Front door", ["CameraEditor.Placeholder.Host"] = "192.168.1.10", ["CameraEditor.Placeholder.OnvifPort"] = "8899", @@ -448,6 +456,7 @@ private static LangCode DetectSystem() ["Settings.Video.TelemetryOverlay"] = "Показывать телеметрию на видео", ["Settings.Video.AutoSdHd"] = "Авто SD/HD (sub в гриде, main на весь экран)", + ["Settings.Video.AiForceCpu"] = "Только CPU для ИИ-аналитики (без GPU-ускорения)", ["Settings.Video.MaxGridSessions"] = "Максимум потоков в гриде", ["Settings.Video.RtspTransport"] = "RTSP-транспорт по умолчанию", ["Settings.Video.NetworkInterface"] = "Сетевой интерфейс (поиск)", @@ -487,6 +496,13 @@ private static LangCode DetectSystem() ["CameraEditor.Quality.Auto"] = "Авто (SD/HD)", ["CameraEditor.Quality.AlwaysHd"] = "Всегда HD (main)", ["CameraEditor.Quality.AlwaysSd"] = "Всегда SD (sub)", + ["CameraEditor.Analytics.Section"] = "ИИ-аналитика", + ["CameraEditor.Analytics.Enable"] = "Включить детекцию объектов", + ["CameraEditor.Analytics.Classes"] = "Классы", + ["CameraEditor.Analytics.Threshold"] = "Порог уверенности", + ["CameraEditor.Analytics.Fps"] = "Кадров/с анализа", + ["CameraEditor.Analytics.AutoRecord"] = "Автозапись по детекции", + ["CameraEditor.Analytics.PostEvent"] = "Дозапись после (с)", ["CameraEditor.Placeholder.Name"] = "Входная", ["CameraEditor.Placeholder.Host"] = "192.168.1.10", ["CameraEditor.Placeholder.OnvifPort"] = "8899", diff --git a/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs index c63352b..5403ce2 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs @@ -10,6 +10,7 @@ using CommunityToolkit.Mvvm.Input; using Microsoft.Extensions.Logging; using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; using OpenIPC.Viewer.Core.Services; using OpenIPC.Viewer.Core.Video; @@ -53,6 +54,16 @@ public sealed partial class CameraEditorViewModel : ViewModelBase [ObservableProperty] private StreamQualityOption? _selectedStreamQuality; + // AI analytics (Phase 15.4). Threshold 0..1; fps clamped 1..10 by the engine. + [ObservableProperty] private bool _aiEnabled; + [ObservableProperty] private double _aiThreshold = 0.5; + [ObservableProperty] private int _aiFps = 3; + [ObservableProperty] private bool _aiAutoRecord; + [ObservableProperty] private int _aiPostEventSeconds = 15; + + // Curated, surveillance-relevant subset of COCO classes the user can toggle. + public ObservableCollection AnalyticsClasses { get; } = new(); + // Includes a leading null entry so the user can pick "no group". public ObservableCollection AvailableGroups { get; } = new(); @@ -74,6 +85,21 @@ public CameraEditorViewModel(IVideoEngine engine, CameraDirectoryService directo _userSettings = userSettings; _logger = logger; SelectedStreamQuality = StreamQualityOptions[0]; // Auto + PopulateAnalyticsClasses(); + } + + // Surveillance-relevant COCO classes. person is pre-selected so a fresh + // enable does something sensible; the user can broaden/narrow from here. + private static readonly (int Id, bool Default)[] CuratedClasses = + { + (0, true), (1, false), (2, false), (3, false), (5, false), (7, false), (15, false), (16, false), + }; + + private void PopulateAnalyticsClasses() + { + if (AnalyticsClasses.Count > 0) return; + foreach (var (id, def) in CuratedClasses) + AnalyticsClasses.Add(new DetectionClassOption(id, CocoClasses.Names[id]) { IsSelected = def }); } public CameraEditorViewModel(Camera existing, CameraCredentials? credentials, CameraCredentials? sshCredentials, IVideoEngine engine, CameraDirectoryService directory, UserSettingsService userSettings, ILogger logger) @@ -94,6 +120,19 @@ public CameraEditorViewModel(Camera existing, CameraCredentials? credentials, Ca _pendingGroupId = existing.GroupId; SelectedStreamQuality = StreamQualityOptions.FirstOrDefault(o => o.Value == existing.StreamQualityOverride) ?? StreamQualityOptions[0]; + + var ai = existing.AnalyticsOrDefault; + AiEnabled = ai.Enabled; + AiThreshold = ai.ConfidenceThreshold; + AiFps = ai.AnalyticsFps; + AiAutoRecord = ai.AutoRecord; + AiPostEventSeconds = ai.PostEventSeconds; + if (ai.ClassIds is { Count: > 0 } ids) + { + var set = new HashSet(ids); + foreach (var opt in AnalyticsClasses) + opt.IsSelected = set.Contains(opt.ClassId); + } } public async Task LoadGroupsAsync(CancellationToken ct) @@ -187,6 +226,7 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe : new CameraCredentials(SshUsername, SshPassword); var quality = SelectedStreamQuality?.Value ?? StreamQualityOverride.Auto; + var analytics = BuildAnalyticsSettings(); if (EditingId is null) { @@ -201,7 +241,8 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe GroupId: SelectedGroup?.Id, StreamQualityOverride: quality, SshCredentials: sshCredentials, - SshPort: sshPort); + SshPort: sshPort, + Analytics: analytics); } else { @@ -216,7 +257,8 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe GroupId: SelectedGroup?.Id, StreamQualityOverride: quality, SshCredentials: sshCredentials, - SshPort: sshPort); + SshPort: sshPort, + Analytics: analytics); } return ok; @@ -298,6 +340,18 @@ private bool TryValidate(out bool ok, out Uri rtspMain, out Uri? rtspSub, out in return true; } + private AnalyticsSettings BuildAnalyticsSettings() + { + var classIds = AnalyticsClasses.Where(c => c.IsSelected).Select(c => c.ClassId).ToArray(); + return new AnalyticsSettings( + Enabled: AiEnabled, + ClassIds: classIds.Length > 0 ? classIds : null, + ConfidenceThreshold: (float)AiThreshold, + AnalyticsFps: AiFps, + AutoRecord: AiAutoRecord, + PostEventSeconds: AiPostEventSeconds); + } + private static RtspTransport ParseTransport(string? s) => s?.ToLowerInvariant() switch { "udp" => RtspTransport.Udp, @@ -305,6 +359,21 @@ private bool TryValidate(out bool ok, out Uri rtspMain, out Uri? rtspSub, out in }; } +// Toggleable detection class in the editor (Phase 15.4). +public sealed partial class DetectionClassOption : ObservableObject +{ + public DetectionClassOption(int classId, string name) + { + ClassId = classId; + Name = name; + } + + public int ClassId { get; } + public string Name { get; } + + [ObservableProperty] private bool _isSelected; +} + public sealed record CameraEditorResult(NewCameraRequest? NewRequest, UpdateCameraRequest? UpdateRequest); // Combo item for the per-camera SD/HD override picker (Phase 12.2). diff --git a/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs index bdff6dc..b6be0cf 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs @@ -31,6 +31,9 @@ public sealed partial class SettingsPageViewModel : ViewModelBase [ObservableProperty] private string _rtspTransport = "tcp"; [ObservableProperty] private bool _autoSdHd = true; + // AI analytics (Phase 15.2). On → pin the CPU execution provider. + [ObservableProperty] private bool _aiForceCpu; + [ObservableProperty] private NetworkInterfaceOption? _selectedNetworkInterface; [ObservableProperty] private string _language = "system"; @@ -150,6 +153,7 @@ private void Load() MaxConcurrentGridSessions = s.MaxConcurrentGridSessions; RtspTransport = s.RtspTransport; AutoSdHd = s.AutoSdHd; + AiForceCpu = string.Equals(s.AiAcceleration, "force-cpu", StringComparison.OrdinalIgnoreCase); SelectedNetworkInterface = NetworkInterfaceOptions.FirstOrDefault(o => o.Value == s.PreferredNetworkInterface) ?? NetworkInterfaceOptions[0]; @@ -170,6 +174,7 @@ private void Load() partial void OnMaxConcurrentGridSessionsChanged(int value) => Persist(); partial void OnRtspTransportChanged(string value) => Persist(); partial void OnAutoSdHdChanged(bool value) => Persist(); + partial void OnAiForceCpuChanged(bool value) => Persist(); partial void OnSelectedNetworkInterfaceChanged(NetworkInterfaceOption? value) => Persist(); partial void OnRecordingsDirOverrideChanged(string value) => Persist(); partial void OnLanguageChanged(string value) => Persist(); @@ -190,6 +195,7 @@ private void Persist() MaxConcurrentGridSessions = MaxConcurrentGridSessions, RtspTransport = RtspTransport, AutoSdHd = AutoSdHd, + AiAcceleration = AiForceCpu ? "force-cpu" : "auto", PreferredNetworkInterface = SelectedNetworkInterface?.Value ?? "", RecordingsDirOverride = RecordingsDirOverride, Language = Language, diff --git a/src/OpenIPC.Viewer.App/Views/Dialogs/CameraEditorContent.axaml b/src/OpenIPC.Viewer.App/Views/Dialogs/CameraEditorContent.axaml index 4322f47..45abd41 100644 --- a/src/OpenIPC.Viewer.App/Views/Dialogs/CameraEditorContent.axaml +++ b/src/OpenIPC.Viewer.App/Views/Dialogs/CameraEditorContent.axaml @@ -126,6 +126,47 @@ HorizontalAlignment="Stretch" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 299881867faadef9f7818a04aac4cc77e0077428 Mon Sep 17 00:00:00 2001 From: keyldev Date: Fri, 19 Jun 2026 22:36:08 +0300 Subject: [PATCH 08/13] feat(analytics): AI control center page (15.7) - AnalyticsPage: engine status + diagnostics, analytics-enabled cameras, recent detection events - 1Hz diagnostics poll bound to the view's visual lifetime - Nav entry (desktop sidebar) + VM registration + DataTemplate; EN/RU strings --- src/OpenIPC.Viewer.App/App.axaml | 3 + src/OpenIPC.Viewer.App/Services/Localizer.cs | 22 ++++ .../ViewModels/AnalyticsPageViewModel.cs | 123 ++++++++++++++++++ .../ViewModels/MainWindowViewModel.cs | 6 + src/OpenIPC.Viewer.App/Views/MainView.axaml | 9 ++ .../Views/Pages/AnalyticsPage.axaml | 101 ++++++++++++++ .../Views/Pages/AnalyticsPage.axaml.cs | 24 ++++ .../SharedComposition.cs | 1 + 8 files changed, 289 insertions(+) create mode 100644 src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs create mode 100644 src/OpenIPC.Viewer.App/Views/Pages/AnalyticsPage.axaml create mode 100644 src/OpenIPC.Viewer.App/Views/Pages/AnalyticsPage.axaml.cs diff --git a/src/OpenIPC.Viewer.App/App.axaml b/src/OpenIPC.Viewer.App/App.axaml index b91c3fd..ebbd58b 100644 --- a/src/OpenIPC.Viewer.App/App.axaml +++ b/src/OpenIPC.Viewer.App/App.axaml @@ -29,6 +29,9 @@ + + + diff --git a/src/OpenIPC.Viewer.App/Services/Localizer.cs b/src/OpenIPC.Viewer.App/Services/Localizer.cs index 9e5f56e..080e314 100644 --- a/src/OpenIPC.Viewer.App/Services/Localizer.cs +++ b/src/OpenIPC.Viewer.App/Services/Localizer.cs @@ -65,7 +65,18 @@ private static LangCode DetectSystem() ["Nav.Recordings"] = "Recordings", ["Nav.RecordingsShort"] = "Records", ["Nav.Events"] = "Events", + ["Nav.Analytics"] = "AI", ["Nav.Settings"] = "Settings", + ["Analytics.Overview"] = "Engine", + ["Analytics.Provider"] = "Execution provider", + ["Analytics.ActiveCameras"] = "Active cameras", + ["Analytics.Processed"] = "Frames processed", + ["Analytics.Dropped"] = "Frames dropped", + ["Analytics.Queue"] = "Queue:", + ["Analytics.Latency"] = "Latency:", + ["Analytics.Cameras"] = "Cameras with detection", + ["Analytics.RecentDetections"] = "Recent detections", + ["Analytics.AllClasses"] = "all classes", ["Common.Cancel"] = "Cancel", ["Common.Delete"] = "Delete", @@ -368,7 +379,18 @@ private static LangCode DetectSystem() ["Nav.Recordings"] = "Записи", ["Nav.RecordingsShort"] = "Записи", ["Nav.Events"] = "События", + ["Nav.Analytics"] = "ИИ", ["Nav.Settings"] = "Настройки", + ["Analytics.Overview"] = "Движок", + ["Analytics.Provider"] = "Провайдер выполнения", + ["Analytics.ActiveCameras"] = "Активные камеры", + ["Analytics.Processed"] = "Кадров обработано", + ["Analytics.Dropped"] = "Кадров отброшено", + ["Analytics.Queue"] = "Очередь:", + ["Analytics.Latency"] = "Задержка:", + ["Analytics.Cameras"] = "Камеры с детекцией", + ["Analytics.RecentDetections"] = "Недавние детекции", + ["Analytics.AllClasses"] = "все классы", ["Common.Cancel"] = "Отмена", ["Common.Delete"] = "Удалить", diff --git a/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs new file mode 100644 index 0000000..1cb6d66 --- /dev/null +++ b/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Avalonia.Threading; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Events; +using OpenIPC.Viewer.Core.Services; + +namespace OpenIPC.Viewer.App.ViewModels; + +// AI control center (Phase 15.7): engine status + diagnostics, the cameras with +// analytics on, and recent detection events. The diagnostics counters poll on a +// 1 Hz timer that the view starts/stops with its visual lifetime. +public sealed partial class AnalyticsPageViewModel : ViewModelBase +{ + private readonly IAnalyticsEngine _engine; + private readonly CameraDirectoryService _directory; + private readonly IEventRepository _events; + private readonly ILogger _logger; + private readonly DispatcherTimer _timer; + + public string Title => Localizer.Instance["Nav.Analytics"]; + + [ObservableProperty] private bool _engineReady; + [ObservableProperty] private string _activeProvider = "—"; + [ObservableProperty] private long _framesProcessed; + [ObservableProperty] private long _framesDropped; + [ObservableProperty] private int _queueDepth; + [ObservableProperty] private double _averageLatencyMs; + [ObservableProperty] private int _activeCameras; + + public ObservableCollection Cameras { get; } = new(); + public ObservableCollection RecentDetections { get; } = new(); + + public AnalyticsPageViewModel( + IAnalyticsEngine engine, + CameraDirectoryService directory, + IEventRepository events, + ILogger logger) + { + _engine = engine; + _directory = directory; + _events = events; + _logger = logger; + _timer = new DispatcherTimer { Interval = TimeSpan.FromSeconds(1) }; + _timer.Tick += (_, _) => RefreshDiagnostics(); + } + + // Called by the view when shown. + public async Task StartAsync() + { + await RefreshAsync(CancellationToken.None).ConfigureAwait(true); + _timer.Start(); + } + + // Called by the view when hidden — stop polling. + public void Stop() => _timer.Stop(); + + [RelayCommand] + private Task RefreshAsync(CancellationToken ct) => ReloadAsync(ct); + + private async Task ReloadAsync(CancellationToken ct) + { + RefreshDiagnostics(); + try + { + var cameras = await _directory.ListAsync(ct).ConfigureAwait(true); + var names = cameras.ToDictionary(c => c.Id, c => c.Name); + + Cameras.Clear(); + foreach (var c in cameras.Where(c => c.AnalyticsOrDefault.Enabled)) + { + var s = c.AnalyticsOrDefault; + Cameras.Add(new AnalyticsCameraRow( + c.Name, ClassesSummary(s.ClassIds), s.AnalyticsFps, s.ConfidenceThreshold, s.AutoRecord)); + } + + var events = await _events.ListAsync(null, EventKind.Detection, null, 50, ct).ConfigureAwait(true); + RecentDetections.Clear(); + foreach (var ev in events) + { + var name = names.TryGetValue(ev.CameraId, out var n) ? n : ev.CameraId.ToString(); + RecentDetections.Add(new DetectionEventRow( + name, ev.OccurredAt.ToLocalTime(), ev.Summary ?? "")); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load analytics control center data."); + } + } + + private void RefreshDiagnostics() + { + var d = _engine.Diagnostics; + EngineReady = _engine.IsReady; + ActiveProvider = _engine.ActiveProvider.ToString(); + FramesProcessed = d.FramesProcessed; + FramesDropped = d.FramesDropped; + QueueDepth = d.QueueDepth; + AverageLatencyMs = d.AverageLatencyMs; + ActiveCameras = d.ActiveCameras; + } + + private static string ClassesSummary(IReadOnlyCollection? ids) + { + if (ids is not { Count: > 0 }) return Localizer.Instance["Analytics.AllClasses"]; + return string.Join(", ", ids.Select(id => + id >= 0 && id < CocoClasses.Names.Count ? CocoClasses.Names[id] : $"class{id}")); + } +} + +public sealed record AnalyticsCameraRow( + string Name, string Classes, int Fps, float Threshold, bool AutoRecord); + +public sealed record DetectionEventRow(string CameraName, DateTime OccurredAt, string Summary); diff --git a/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs index a50983f..d1d93ec 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs @@ -23,6 +23,7 @@ public sealed partial class MainWindowViewModel : ViewModelBase, IRecipient CurrentPage is CameraLibraryPageViewModel or SingleCameraPageViewModel; public bool IsRecordingsSelected => CurrentPage is RecordingsPageViewModel; public bool IsEventsSelected => CurrentPage is EventsPageViewModel; + public bool IsAnalyticsSelected => CurrentPage is AnalyticsPageViewModel; public bool IsSettingsSelected => CurrentPage is SettingsPageViewModel; public MainWindowViewModel( @@ -52,6 +55,7 @@ public MainWindowViewModel( CameraLibraryPageViewModel library, RecordingsPageViewModel recordings, EventsPageViewModel events, + AnalyticsPageViewModel analytics, SettingsPageViewModel settings, CameraDirectoryService directory, SingleCameraPageFactory singleCameraFactory, @@ -61,6 +65,7 @@ public MainWindowViewModel( Library = library; Recordings = recordings; Events = events; + Analytics = analytics; Settings = settings; _directory = directory; _singleCameraFactory = singleCameraFactory; @@ -113,6 +118,7 @@ private void Navigate(string target) "library" => Library, "recordings" => Recordings, "events" => Events, + "analytics" => Analytics, "settings" => Settings, _ => CurrentPage, }; diff --git a/src/OpenIPC.Viewer.App/Views/MainView.axaml b/src/OpenIPC.Viewer.App/Views/MainView.axaml index e43d0ff..1e1f823 100644 --- a/src/OpenIPC.Viewer.App/Views/MainView.axaml +++ b/src/OpenIPC.Viewer.App/Views/MainView.axaml @@ -81,6 +81,15 @@ + + + + + + + + + + + + +