diff --git a/Directory.Packages.props b/Directory.Packages.props index e104129..829720f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -64,6 +64,13 @@ + + + diff --git a/OpenIPC.Viewer.slnx b/OpenIPC.Viewer.slnx index 4497995..12ba2b6 100644 --- a/OpenIPC.Viewer.slnx +++ b/OpenIPC.Viewer.slnx @@ -3,6 +3,7 @@ + @@ -13,8 +14,11 @@ + - + + + diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 98082b7..a5bf8e3 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -56,7 +56,7 @@ and scope live in the planning docs (`dashboard-ideas-roadmap-ru.md`). | 12 | Streaming hardening | Smart-pause hidden tiles, auto SD/HD, watchdog + backoff, last-frame hold, error tile | ✅ Done | | 13 | SSH device suite | SSH terminal, SCP file manager, open-in-browser, config push | ✅ Done | | 14 | Snapshots & viewer | Always-HD snapshot, snapshot browser, built-in viewer + basic editor | ✅ Done | -| 15 | Local AI analytics | ONNX object detection per camera, auto-record, control center, CPU fallback | 📋 Planned | +| 15 | Local AI analytics | ONNX object detection per camera, auto-record, control center, CPU fallback | ✅ Done | | 16 | Archive pro | Fragmented MP4, activity calendar, timeline zoom, clip export | 📋 Planned | | 17 | Community & app-level | Tabbed layouts, config export/import, notifications, white-label, issue reporter, RBAC | 📋 Planned | | 18 | Streq remote access | Cloud multistreaming across devices: LAN/overlay/relay routing, enrollment, WebRTC/HLS, cross-device sync | 📋 Planned | diff --git a/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs b/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs new file mode 100644 index 0000000..e8bda19 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ExecutionProviderChain.cs @@ -0,0 +1,30 @@ +using System; +using System.Collections.Generic; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// The ordered list of execution providers to try for the current platform, +// most-preferred first. CPU is always the implicit final fallback (the base +// ONNX Runtime package registers it on every RID) so it is not listed here. +// +// Only providers reachable through the generic AppendExecutionProvider(string) +// API are listed — they no-op/throw gracefully when their native EP isn't +// compiled into the referenced package, and we catch + fall back. DirectML and +// CUDA need their dedicated packages + AppendExecutionProvider_DML/_CUDA calls +// wired in the platform composition; until those are added per head, Windows +// and Linux run on CPU. +internal static class ExecutionProviderChain +{ + public static IReadOnlyList<(string Name, ExecutionProvider Provider)> ForCurrentPlatform() + { + if (OperatingSystem.IsAndroid()) + return new[] { ("NNAPI", ExecutionProvider.NnApi), ("XNNPACK", ExecutionProvider.Xnnpack) }; + + if (OperatingSystem.IsIOS() || OperatingSystem.IsMacOS() || OperatingSystem.IsMacCatalyst()) + return new[] { ("CoreML", ExecutionProvider.CoreMl) }; + + // Windows / Linux: CPU only for now (GPU EPs are a per-head package job). + return Array.Empty<(string, ExecutionProvider)>(); + } +} diff --git a/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs b/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs new file mode 100644 index 0000000..8e586c4 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ModelCatalog.cs @@ -0,0 +1,30 @@ +using System; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// A downloadable detection model: where to fetch it, its integrity hash, and +// how to build the ModelSpec once it is on disk. +public sealed record ModelDescriptor( + string Name, + string FileName, + Uri? DownloadUri, + string? Sha256Hex, + Func CreateSpec); + +// Known models. We ship Apache-2.0 YOLOX (never AGPL Ultralytics YOLO) to stay +// MIT/store compatible. The model is an asset fetched on first enable, not +// checked into the repo. +public static class ModelCatalog +{ + // Official YOLOX-tiny ONNX export from the upstream release assets + // (~20 MB). SHA-256 verified against the 0.1.1rc0 asset on 2026-06-19. + public static ModelDescriptor YoloxTiny { get; } = new( + Name: "YOLOX-tiny", + FileName: "yolox_tiny.onnx", + DownloadUri: new Uri("https://github.com/Megvii-BaseDetection/YOLOX/releases/download/0.1.1rc0/yolox_tiny.onnx"), + Sha256Hex: "427CC366D34E27FF7A03E2899B5E3671425C262EA2291F88BB942BC1CC70B0F7", + CreateSpec: ModelSpec.YoloxTiny); + + public static ModelDescriptor Default => YoloxTiny; +} diff --git a/src/OpenIPC.Viewer.Analytics/ModelProvider.cs b/src/OpenIPC.Viewer.Analytics/ModelProvider.cs new file mode 100644 index 0000000..b6454cd --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ModelProvider.cs @@ -0,0 +1,109 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Platform; + +namespace OpenIPC.Viewer.Analytics; + +// Resolves the detection model file (Phase 15 — download on first enable, cache +// in AppData, verify integrity, repo stays binary-free). Resolution order: +// 1. OPENIPC_DETECTION_MODEL env var (explicit local override / CI fixture), +// 2. cached file in {AppData}/models (re-verified if a hash is pinned), +// 3. download from the descriptor URI into the cache. +public sealed class ModelProvider : IModelProvider +{ + private readonly IFileSystem _fs; + private readonly ILogger _log; + private readonly ModelDescriptor _descriptor; + private readonly Func _httpFactory; + private readonly SemaphoreSlim _gate = new(1, 1); + + public ModelProvider(IFileSystem fs, ILogger log) + : this(fs, log, ModelCatalog.Default, () => new HttpClient { Timeout = TimeSpan.FromMinutes(5) }) + { + } + + internal ModelProvider(IFileSystem fs, ILogger log, + ModelDescriptor descriptor, Func httpFactory) + { + _fs = fs; + _log = log; + _descriptor = descriptor; + _httpFactory = httpFactory; + } + + public async Task EnsureModelAsync(CancellationToken ct) + { + var overridePath = Environment.GetEnvironmentVariable("OPENIPC_DETECTION_MODEL"); + if (!string.IsNullOrWhiteSpace(overridePath) && File.Exists(overridePath)) + { + _log.LogInformation("Using detection model override: {Path}", overridePath); + return _descriptor.CreateSpec(overridePath); + } + + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + var modelsDir = Directory.CreateDirectory(Path.Combine(_fs.AppDataDir.FullName, "models")); + var target = Path.Combine(modelsDir.FullName, _descriptor.FileName); + + if (File.Exists(target) && await VerifyAsync(target, ct).ConfigureAwait(false)) + return _descriptor.CreateSpec(target); + + if (_descriptor.DownloadUri is null) + throw new InvalidOperationException( + $"Model {_descriptor.Name} is missing and no download URI is configured."); + + await DownloadAsync(_descriptor.DownloadUri, target, ct).ConfigureAwait(false); + + if (!await VerifyAsync(target, ct).ConfigureAwait(false)) + { + File.Delete(target); + throw new InvalidOperationException( + $"Downloaded model {_descriptor.Name} failed its integrity check."); + } + + return _descriptor.CreateSpec(target); + } + finally + { + _gate.Release(); + } + } + + private async Task VerifyAsync(string path, CancellationToken ct) + { + if (_descriptor.Sha256Hex is null) return true; // not pinned yet + using var sha = SHA256.Create(); + await using var stream = File.OpenRead(path); + var hash = await sha.ComputeHashAsync(stream, ct).ConfigureAwait(false); + var hex = Convert.ToHexString(hash); + var ok = string.Equals(hex, _descriptor.Sha256Hex, StringComparison.OrdinalIgnoreCase); + if (!ok) + _log.LogWarning("Model {Name} sha256 mismatch (expected {Expected}, got {Actual}).", + _descriptor.Name, _descriptor.Sha256Hex, hex); + return ok; + } + + private async Task DownloadAsync(Uri uri, string target, CancellationToken ct) + { + _log.LogInformation("Downloading detection model {Name} from {Uri}", _descriptor.Name, uri); + using var http = _httpFactory(); + using var resp = await http.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false); + resp.EnsureSuccessStatusCode(); + + var tmp = target + ".part"; + await using (var dst = File.Create(tmp)) + await using (var src = await resp.Content.ReadAsStreamAsync(ct).ConfigureAwait(false)) + { + await src.CopyToAsync(dst, ct).ConfigureAwait(false); + } + File.Move(tmp, target, overwrite: true); + _log.LogInformation("Detection model {Name} cached at {Path}", _descriptor.Name, target); + } +} diff --git a/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs b/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs new file mode 100644 index 0000000..75d3604 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/ObjectDetectionEngine.cs @@ -0,0 +1,262 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Video; +using ReactiveSubject = System.Reactive.Subjects.Subject; + +namespace OpenIPC.Viewer.Analytics; + +// The Phase 15.3 sampling pipeline. Per camera: a FrameSampler thins the +// decoder's ~25 FPS down to analyticsFps, each kept frame is downscaled + +// copied (the decoder owns its buffer) and pushed into a small bounded channel +// with drop-oldest semantics. A single worker drains the channel and runs the +// shared detector, so memory/CPU stay bounded regardless of camera count. +public sealed class ObjectDetectionEngine : IAnalyticsEngine +{ + private const int QueueCapacity = 8; + private const int DownscaleMaxSide = 640; + + private readonly IObjectDetector _detector; + private readonly IModelProvider _modelProvider; + private readonly ILogger _log; + + private readonly ConcurrentDictionary _cameras = new(); + private readonly Channel _queue = Channel.CreateBounded( + new BoundedChannelOptions(QueueCapacity) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + }); + private readonly ReactiveSubject _results = new(); + private readonly CancellationTokenSource _shutdown = new(); + private readonly object _statsLock = new(); + private readonly SemaphoreSlim _initGate = new(1, 1); + + private long _sampled; + private long _processed; + private long _latencyCount; + private double _latencySumMs; + private Task? _worker; + private bool _initialized; + + public ObjectDetectionEngine(IObjectDetector detector, IModelProvider modelProvider, + ILogger log) + { + _detector = detector; + _modelProvider = modelProvider; + _log = log; + } + + public bool IsReady => _detector.IsLoaded; + public ExecutionProvider ActiveProvider => _detector.ActiveProvider; + public AnalyticsEngineStatus Status { get; private set; } = AnalyticsEngineStatus.NotStarted; + public IObservable Results => _results; + + public AnalyticsDiagnostics Diagnostics + { + get + { + long sampled = Interlocked.Read(ref _sampled); + long processed = Interlocked.Read(ref _processed); + double avg; + lock (_statsLock) + avg = _latencyCount == 0 ? 0 : _latencySumMs / _latencyCount; + return new AnalyticsDiagnostics( + ActiveCameras: _cameras.Count, + FramesSampled: sampled, + FramesProcessed: processed, + FramesDropped: Math.Max(0, sampled - processed - QueueDepth()), + QueueDepth: QueueDepth(), + AverageLatencyMs: avg); + } + } + + public async Task InitializeAsync(AiAcceleration acceleration, CancellationToken ct) + { + await _initGate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_initialized) return; + Status = AnalyticsEngineStatus.Preparing; + var spec = await _modelProvider.EnsureModelAsync(ct).ConfigureAwait(false); + Status = AnalyticsEngineStatus.Loading; + await _detector.LoadAsync(spec, acceleration, ct).ConfigureAwait(false); + _worker = Task.Run(() => WorkerLoopAsync(_shutdown.Token)); + _initialized = true; + Status = AnalyticsEngineStatus.Ready; + _log.LogInformation("Analytics engine ready on {Provider}.", _detector.ActiveProvider); + } + catch + { + Status = AnalyticsEngineStatus.Failed; + throw; + } + finally + { + _initGate.Release(); + } + } + + public void Attach(CameraId cameraId, IObservable frames, + Func settings, Func isActive) + { + Detach(cameraId); + var reg = new CameraRegistration(new FrameSampler(settings().AnalyticsFps), settings, isActive); + reg.Subscription = frames.Subscribe(new FrameObserver(this, cameraId, reg)); + _cameras[cameraId] = reg; + } + + public void Detach(CameraId cameraId) + { + if (_cameras.TryRemove(cameraId, out var reg)) + reg.Dispose(); + } + + private void OnFrame(CameraId cameraId, CameraRegistration reg, in VideoFrame frame) + { + var settings = reg.Settings(); + if (!settings.Enabled || !reg.IsActive()) return; + + // Keep the sampler in step with a live FPS change. + if (reg.Sampler.TargetFps != ClampFps(settings.AnalyticsFps)) + reg.Sampler = new FrameSampler(settings.AnalyticsFps); + + if (!reg.Sampler.ShouldSample(frame.ReceivedAt)) return; + + var buffer = Downscale(frame, DownscaleMaxSide); + if (_queue.Writer.TryWrite(new WorkItem(cameraId, buffer, settings, frame.ReceivedAt))) + Interlocked.Increment(ref _sampled); + } + + private async Task WorkerLoopAsync(CancellationToken ct) + { + try + { + await foreach (var item in _queue.Reader.ReadAllAsync(ct).ConfigureAwait(false)) + { + if (!_detector.IsLoaded) continue; + + try + { + var sw = Stopwatch.StartNew(); + var detections = _detector.Detect(item.Frame, item.Settings.ToDetectOptions()); + sw.Stop(); + + Interlocked.Increment(ref _processed); + lock (_statsLock) + { + _latencySumMs += sw.Elapsed.TotalMilliseconds; + _latencyCount++; + } + + _results.OnNext(new DetectionResult( + item.CameraId, item.Timestamp, detections, sw.Elapsed.TotalMilliseconds)); + } + catch (Exception ex) + { + _log.LogWarning(ex, "Inference failed for camera {Camera}.", item.CameraId); + } + } + } + catch (OperationCanceledException) + { + // shutting down + } + } + + private int QueueDepth() => _queue.Reader.Count; + + private static int ClampFps(int fps) => fps < 1 ? 1 : fps > 30 ? 30 : fps; + + // Full-frame aspect-preserving downscale into a tightly packed BGRA buffer. + // Because it is the WHOLE frame (no crop), detection coords normalized to + // this buffer equal those of the original — the overlay maps cleanly. + private static FrameBuffer Downscale(in VideoFrame f, int maxSide) + { + var scale = Math.Min(1f, (float)maxSide / Math.Max(f.Width, f.Height)); + var w = Math.Max(1, (int)(f.Width * scale)); + var h = Math.Max(1, (int)(f.Height * scale)); + var dst = new byte[w * h * 4]; + var sxStep = (float)f.Width / w; + var syStep = (float)f.Height / h; + + for (var y = 0; y < h; y++) + { + var srcRow = Math.Min(f.Height - 1, (int)(y * syStep)) * f.Stride; + var dstRow = y * w * 4; + for (var x = 0; x < w; x++) + { + var so = srcRow + Math.Min(f.Width - 1, (int)(x * sxStep)) * 4; + var doff = dstRow + x * 4; + dst[doff] = f.Bgra[so]; + dst[doff + 1] = f.Bgra[so + 1]; + dst[doff + 2] = f.Bgra[so + 2]; + dst[doff + 3] = 255; + } + } + + return new FrameBuffer(dst, w, h, w * 4); + } + + public async ValueTask DisposeAsync() + { + _shutdown.Cancel(); + _queue.Writer.TryComplete(); + foreach (var reg in _cameras.Values) reg.Dispose(); + _cameras.Clear(); + if (_worker is not null) + { + try { await _worker.ConfigureAwait(false); } + catch (OperationCanceledException) { } + } + _results.OnCompleted(); + _results.Dispose(); + await _detector.DisposeAsync().ConfigureAwait(false); + _shutdown.Dispose(); + } + + private sealed record WorkItem(CameraId CameraId, FrameBuffer Frame, AnalyticsSettings Settings, DateTime Timestamp); + + private sealed class CameraRegistration : IDisposable + { + public CameraRegistration(FrameSampler sampler, Func settings, Func isActive) + { + Sampler = sampler; + Settings = settings; + IsActive = isActive; + } + + public FrameSampler Sampler { get; set; } + public Func Settings { get; } + public Func IsActive { get; } + public IDisposable? Subscription { get; set; } + + public void Dispose() => Subscription?.Dispose(); + } + + // Lightweight IObserver so we don't depend on Rx subscription extensions. + private sealed class FrameObserver : IObserver + { + private readonly ObjectDetectionEngine _engine; + private readonly CameraId _cameraId; + private readonly CameraRegistration _reg; + + public FrameObserver(ObjectDetectionEngine engine, CameraId cameraId, CameraRegistration reg) + { + _engine = engine; + _cameraId = cameraId; + _reg = reg; + } + + public void OnNext(VideoFrame value) => _engine.OnFrame(_cameraId, _reg, value); + public void OnError(Exception error) { } + public void OnCompleted() { } + } +} diff --git a/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs b/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs new file mode 100644 index 0000000..868fc19 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/OnnxObjectDetector.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.Analytics; + +// ONNX Runtime object detector (Phase 15.1/15.2). Loads a YOLOX model, selects +// the platform execution provider with a mandatory CPU fallback, and runs +// synchronous inference (the sampling stage owns the worker thread). Not +// thread-safe across concurrent Detect calls — one session, one caller. +public sealed class OnnxObjectDetector : IObjectDetector +{ + private readonly ILogger _log; + private readonly object _gate = new(); + + private InferenceSession? _session; + private ModelSpec? _spec; + private string? _inputName; + + public OnnxObjectDetector(ILogger log) => _log = log; + + public bool IsLoaded => _session is not null; + public ExecutionProvider ActiveProvider { get; private set; } = ExecutionProvider.Cpu; + + public async Task LoadAsync(ModelSpec model, AiAcceleration acceleration, CancellationToken ct) + { + if (model is null) throw new ArgumentNullException(nameof(model)); + + var (session, provider) = await Task.Run(() => CreateSession(model, acceleration), ct) + .ConfigureAwait(false); + + lock (_gate) + { + _session?.Dispose(); + _session = session; + _spec = model; + _inputName = FirstInputName(session); + ActiveProvider = provider; + } + + _log.LogInformation("Detector loaded: {Model} on {Provider} (input {Name})", + model.Name, provider, _inputName); + } + + private (InferenceSession, ExecutionProvider) CreateSession(ModelSpec model, AiAcceleration acceleration) + { + var chain = acceleration == AiAcceleration.ForceCpu + ? Array.Empty<(string Name, ExecutionProvider Provider)>() + : ExecutionProviderChain.ForCurrentPlatform(); + + foreach (var (name, provider) in chain) + { + try + { + var options = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; + options.AppendExecutionProvider(name); + var session = new InferenceSession(model.FilePath, options); + return (session, provider); + } + catch (Exception ex) + { + _log.LogInformation("Execution provider {Provider} unavailable ({Error}); trying next.", + provider, ex.Message); + } + } + + // CPU is always available — this is the guaranteed fallback path. + var cpuOptions = new SessionOptions { GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_ALL }; + return (new InferenceSession(model.FilePath, cpuOptions), ExecutionProvider.Cpu); + } + + public IReadOnlyList Detect(FrameBuffer frame, DetectOptions options) + { + if (frame is null) throw new ArgumentNullException(nameof(frame)); + + InferenceSession session; + ModelSpec spec; + string inputName; + lock (_gate) + { + if (_session is null || _spec is null || _inputName is null) + throw new InvalidOperationException("Detector is not loaded. Call LoadAsync first."); + session = _session; + spec = _spec; + inputName = _inputName; + } + + var letterbox = new LetterboxTransform(frame.Width, frame.Height, spec.InputWidth, spec.InputHeight); + var input = BuildInputTensor(frame, letterbox, spec.InputWidth, spec.InputHeight); + + var inputs = new[] { NamedOnnxValue.CreateFromTensor(inputName, input) }; + using var results = session.Run(inputs); + var output = results[0].AsTensor().ToArray(); + + var raw = YoloxPostProcessor.Process( + output, spec.ClassCount, spec.InputWidth, spec.InputHeight, + spec.Strides, spec.GridDecodeRequired, options); + + var detections = new List(raw.Count); + foreach (var r in raw) + { + var name = r.ClassId >= 0 && r.ClassId < spec.ClassNames.Count + ? spec.ClassNames[r.ClassId] + : $"class{r.ClassId}"; + detections.Add(letterbox.MapToSource(r.ClassId, name, r.Confidence, r.X, r.Y, r.Width, r.Height)); + } + + return detections; + } + + // BGRA source -> CHW float tensor in YOLOX layout: BGR channel order, raw + // 0..255 values (no /255 — the YOLOX export folds normalization in), padded + // with 114, aspect-preserving (nearest-neighbour resize keeps this cheap). + private static DenseTensor BuildInputTensor( + FrameBuffer frame, LetterboxTransform lb, int inputW, int inputH) + { + var tensor = new DenseTensor(new[] { 1, 3, inputH, inputW }); + var buf = tensor.Buffer.Span; + buf.Fill(114f); + + var plane = inputH * inputW; + var dstX0 = (int)MathF.Round(lb.PadX); + var dstY0 = (int)MathF.Round(lb.PadY); + var scaledW = (int)MathF.Round(frame.Width * lb.Scale); + var scaledH = (int)MathF.Round(frame.Height * lb.Scale); + var invScale = 1f / lb.Scale; + + for (var dy = 0; dy < scaledH; dy++) + { + var outY = dstY0 + dy; + if (outY < 0 || outY >= inputH) continue; + var sy = Math.Min(frame.Height - 1, (int)(dy * invScale)); + + for (var dx = 0; dx < scaledW; dx++) + { + var outX = dstX0 + dx; + if (outX < 0 || outX >= inputW) continue; + var sx = Math.Min(frame.Width - 1, (int)(dx * invScale)); + + var so = sy * frame.Stride + sx * 4; + var p = outY * inputW + outX; + buf[p] = frame.Bgra[so]; // B + buf[plane + p] = frame.Bgra[so + 1]; // G + buf[2 * plane + p] = frame.Bgra[so + 2]; // R + } + } + + return tensor; + } + + private static string FirstInputName(InferenceSession session) + { + foreach (var key in session.InputMetadata.Keys) + return key; + throw new InvalidOperationException("Model has no inputs."); + } + + public ValueTask DisposeAsync() + { + lock (_gate) + { + _session?.Dispose(); + _session = null; + } + return default; + } +} diff --git a/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj new file mode 100644 index 0000000..2935bf3 --- /dev/null +++ b/src/OpenIPC.Viewer.Analytics/OpenIPC.Viewer.Analytics.csproj @@ -0,0 +1,17 @@ + + + + net9.0 + + + + + + + + + + + + + diff --git a/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj b/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj index d5f28e5..b2c8e88 100644 --- a/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj +++ b/src/OpenIPC.Viewer.Android/OpenIPC.Viewer.Android.csproj @@ -27,8 +27,15 @@ verification, which we don't perform — credentials are AES-GCM via EncryptedFileSecretsStore, RTSP auth is HTTP digest. Revisit when MS ships a patched servicing release. + + XA0141 suppression: ONNX Runtime's android native (libonnxruntime4j_jni.so, + pulled transitively by the Phase 15 analytics engine) isn't yet aligned to + the 16 KB page size Android 16 will require. It's an upstream packaging + advisory, not our code; on-device analytics on Android is a later polish + item (capability-flagged, desktop-first). Revisit when ORT ships aligned + mobile binaries. --> - $(NoWarn);NU1903 + $(NoWarn);NU1903;XA0141 diff --git a/src/OpenIPC.Viewer.App/App.axaml b/src/OpenIPC.Viewer.App/App.axaml index b91c3fd..ebbd58b 100644 --- a/src/OpenIPC.Viewer.App/App.axaml +++ b/src/OpenIPC.Viewer.App/App.axaml @@ -29,6 +29,9 @@ + + + diff --git a/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs b/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs new file mode 100644 index 0000000..e2d67a6 --- /dev/null +++ b/src/OpenIPC.Viewer.App/Controls/DetectionOverlay.cs @@ -0,0 +1,81 @@ +using System.Collections.Generic; +using System.Globalization; +using Avalonia; +using Avalonia.Controls; +using Avalonia.Media; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.App.Controls; + +// Draws detection boxes + labels over the video tile (Phase 15.5). Boxes are +// normalized 0..1 in the source frame, so we just scale them to Bounds — no +// dependency on the model input size. Rendered directly (a Canvas of one box +// per visual would churn the tree); we repaint on each new detection result. +public sealed class DetectionOverlay : Control +{ + public static readonly StyledProperty?> DetectionsProperty = + AvaloniaProperty.Register?>(nameof(Detections)); + + public static readonly StyledProperty ShowBoxesProperty = + AvaloniaProperty.Register(nameof(ShowBoxes), defaultValue: true); + + // Distinct hues per class id so different objects read apart at a glance. + private static readonly Color[] Palette = + { + Color.FromRgb(0x4C, 0xAF, 0x50), Color.FromRgb(0x21, 0x96, 0xF3), + Color.FromRgb(0xFF, 0x98, 0x00), Color.FromRgb(0xE0, 0x40, 0x40), + Color.FromRgb(0x9C, 0x27, 0xB0), Color.FromRgb(0x00, 0xBC, 0xD4), + Color.FromRgb(0xFF, 0xEB, 0x3B), Color.FromRgb(0xFF, 0x40, 0x81), + }; + + static DetectionOverlay() + { + AffectsRender(DetectionsProperty, ShowBoxesProperty); + } + + public IReadOnlyList? Detections + { + get => GetValue(DetectionsProperty); + set => SetValue(DetectionsProperty, value); + } + + public bool ShowBoxes + { + get => GetValue(ShowBoxesProperty); + set => SetValue(ShowBoxesProperty, value); + } + + public override void Render(DrawingContext context) + { + base.Render(context); + var detections = Detections; + if (!ShowBoxes || detections is null || detections.Count == 0) return; + + var w = Bounds.Width; + var h = Bounds.Height; + if (w <= 0 || h <= 0) return; + + foreach (var d in detections) + { + var color = Palette[((d.ClassId % Palette.Length) + Palette.Length) % Palette.Length]; + var pen = new Pen(new SolidColorBrush(color), 2); + + var x = d.X * w; + var y = d.Y * h; + var bw = d.Width * w; + var bh = d.Height * h; + var box = new Rect(x, y, bw, bh); + context.DrawRectangle(null, pen, box); + + var label = $"{d.ClassName} {d.Confidence:0.00}"; + var text = new FormattedText(label, CultureInfo.InvariantCulture, FlowDirection.LeftToRight, + Typeface.Default, 11, Brushes.White); + + var labelW = text.Width + 8; + var labelH = text.Height + 2; + var labelY = y - labelH >= 0 ? y - labelH : y; // flip below the top edge if clipped + context.DrawRectangle(new SolidColorBrush(color), null, new Rect(x, labelY, labelW, labelH)); + context.DrawText(text, new Point(x + 4, labelY + 1)); + } + } +} diff --git a/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs b/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs new file mode 100644 index 0000000..5dba893 --- /dev/null +++ b/src/OpenIPC.Viewer.App/Services/AnalyticsBootstrap.cs @@ -0,0 +1,60 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.Core.Analytics; + +namespace OpenIPC.Viewer.App.Services; + +// Lazily brings the analytics engine online (Phase 15). The first tile with +// analytics enabled calls EnsureStartedAsync, which downloads/loads the model, +// picks the execution provider, and starts the auto-record coordinator. Best- +// effort: a failure (e.g. offline first run, no model) is logged and analytics +// simply stays off — it never blocks the UI or crashes the app. +public sealed class AnalyticsBootstrap +{ + private readonly IAnalyticsEngine _engine; + private readonly AutoRecordCoordinator _autoRecord; + private readonly UserSettingsService _settings; + private readonly ILogger _log; + private readonly SemaphoreSlim _gate = new(1, 1); + private bool _started; + + public AnalyticsBootstrap( + IAnalyticsEngine engine, + AutoRecordCoordinator autoRecord, + UserSettingsService settings, + ILogger log) + { + _engine = engine; + _autoRecord = autoRecord; + _settings = settings; + _log = log; + _autoRecord.Failed += (_, ex) => _log.LogWarning(ex, "Auto-record error."); + } + + public bool IsReady => _engine.IsReady; + + public async Task EnsureStartedAsync() + { + if (_started) return; + await _gate.WaitAsync().ConfigureAwait(false); + try + { + if (_started) return; + await _engine.InitializeAsync(_settings.AiAcceleration, CancellationToken.None).ConfigureAwait(false); + _autoRecord.Start(); + _started = true; + _log.LogInformation("Analytics engine started on {Provider}.", _engine.ActiveProvider); + } + catch (Exception ex) + { + // Leave _started false so a later enable retries (e.g. once online). + _log.LogError(ex, "Analytics engine failed to start; analytics stays off."); + } + finally + { + _gate.Release(); + } + } +} diff --git a/src/OpenIPC.Viewer.App/Services/Localizer.cs b/src/OpenIPC.Viewer.App/Services/Localizer.cs index c265f6c..314e107 100644 --- a/src/OpenIPC.Viewer.App/Services/Localizer.cs +++ b/src/OpenIPC.Viewer.App/Services/Localizer.cs @@ -65,7 +65,24 @@ private static LangCode DetectSystem() ["Nav.Recordings"] = "Recordings", ["Nav.RecordingsShort"] = "Records", ["Nav.Events"] = "Events", + ["Nav.Analytics"] = "AI", ["Nav.Settings"] = "Settings", + ["Analytics.Overview"] = "Engine", + ["Analytics.Status"] = "Status", + ["Analytics.Status.NotStarted"] = "Not started", + ["Analytics.Status.Preparing"] = "Downloading model…", + ["Analytics.Status.Loading"] = "Loading model…", + ["Analytics.Status.Ready"] = "Ready", + ["Analytics.Status.Failed"] = "Failed (see logs)", + ["Analytics.Provider"] = "Execution provider", + ["Analytics.ActiveCameras"] = "Active cameras", + ["Analytics.Processed"] = "Frames processed", + ["Analytics.Dropped"] = "Frames dropped", + ["Analytics.Queue"] = "Queue:", + ["Analytics.Latency"] = "Latency:", + ["Analytics.Cameras"] = "Cameras with detection", + ["Analytics.RecentDetections"] = "Recent detections", + ["Analytics.AllClasses"] = "all classes", ["Common.Cancel"] = "Cancel", ["Common.Delete"] = "Delete", @@ -153,6 +170,7 @@ private static LangCode DetectSystem() ["Settings.Video.TelemetryOverlay"] = "Show telemetry overlay on live view", ["Settings.Video.AutoSdHd"] = "Auto SD/HD (sub in grid, main when full-screen)", + ["Settings.Video.AiForceCpu"] = "Force CPU for AI analytics (no GPU acceleration)", ["Settings.Video.MaxGridSessions"] = "Max concurrent grid sessions", ["Settings.Video.RtspTransport"] = "Default RTSP transport", ["Settings.Video.NetworkInterface"] = "Network interface (discovery)", @@ -192,6 +210,13 @@ private static LangCode DetectSystem() ["CameraEditor.Quality.Auto"] = "Auto (SD/HD)", ["CameraEditor.Quality.AlwaysHd"] = "Always HD (main)", ["CameraEditor.Quality.AlwaysSd"] = "Always SD (sub)", + ["CameraEditor.Analytics.Section"] = "AI analytics", + ["CameraEditor.Analytics.Enable"] = "Enable object detection", + ["CameraEditor.Analytics.Classes"] = "Classes", + ["CameraEditor.Analytics.Threshold"] = "Confidence threshold", + ["CameraEditor.Analytics.Fps"] = "Analysis FPS", + ["CameraEditor.Analytics.AutoRecord"] = "Auto-record on detection", + ["CameraEditor.Analytics.PostEvent"] = "Keep recording after (s)", ["CameraEditor.Placeholder.Name"] = "Front door", ["CameraEditor.Placeholder.Host"] = "192.168.1.10", ["CameraEditor.Placeholder.OnvifPort"] = "8899", @@ -360,7 +385,24 @@ private static LangCode DetectSystem() ["Nav.Recordings"] = "Записи", ["Nav.RecordingsShort"] = "Записи", ["Nav.Events"] = "События", + ["Nav.Analytics"] = "ИИ", ["Nav.Settings"] = "Настройки", + ["Analytics.Overview"] = "Движок", + ["Analytics.Status"] = "Статус", + ["Analytics.Status.NotStarted"] = "Не запущен", + ["Analytics.Status.Preparing"] = "Скачивание модели…", + ["Analytics.Status.Loading"] = "Загрузка модели…", + ["Analytics.Status.Ready"] = "Готов", + ["Analytics.Status.Failed"] = "Ошибка (см. логи)", + ["Analytics.Provider"] = "Провайдер выполнения", + ["Analytics.ActiveCameras"] = "Активные камеры", + ["Analytics.Processed"] = "Кадров обработано", + ["Analytics.Dropped"] = "Кадров отброшено", + ["Analytics.Queue"] = "Очередь:", + ["Analytics.Latency"] = "Задержка:", + ["Analytics.Cameras"] = "Камеры с детекцией", + ["Analytics.RecentDetections"] = "Недавние детекции", + ["Analytics.AllClasses"] = "все классы", ["Common.Cancel"] = "Отмена", ["Common.Delete"] = "Удалить", @@ -448,6 +490,7 @@ private static LangCode DetectSystem() ["Settings.Video.TelemetryOverlay"] = "Показывать телеметрию на видео", ["Settings.Video.AutoSdHd"] = "Авто SD/HD (sub в гриде, main на весь экран)", + ["Settings.Video.AiForceCpu"] = "Только CPU для ИИ-аналитики (без GPU-ускорения)", ["Settings.Video.MaxGridSessions"] = "Максимум потоков в гриде", ["Settings.Video.RtspTransport"] = "RTSP-транспорт по умолчанию", ["Settings.Video.NetworkInterface"] = "Сетевой интерфейс (поиск)", @@ -487,6 +530,13 @@ private static LangCode DetectSystem() ["CameraEditor.Quality.Auto"] = "Авто (SD/HD)", ["CameraEditor.Quality.AlwaysHd"] = "Всегда HD (main)", ["CameraEditor.Quality.AlwaysSd"] = "Всегда SD (sub)", + ["CameraEditor.Analytics.Section"] = "ИИ-аналитика", + ["CameraEditor.Analytics.Enable"] = "Включить детекцию объектов", + ["CameraEditor.Analytics.Classes"] = "Классы", + ["CameraEditor.Analytics.Threshold"] = "Порог уверенности", + ["CameraEditor.Analytics.Fps"] = "Кадров/с анализа", + ["CameraEditor.Analytics.AutoRecord"] = "Автозапись по детекции", + ["CameraEditor.Analytics.PostEvent"] = "Дозапись после (с)", ["CameraEditor.Placeholder.Name"] = "Входная", ["CameraEditor.Placeholder.Host"] = "192.168.1.10", ["CameraEditor.Placeholder.OnvifPort"] = "8899", diff --git a/src/OpenIPC.Viewer.App/Services/UserSettings.cs b/src/OpenIPC.Viewer.App/Services/UserSettings.cs index 99fdf89..b05fa58 100644 --- a/src/OpenIPC.Viewer.App/Services/UserSettings.cs +++ b/src/OpenIPC.Viewer.App/Services/UserSettings.cs @@ -32,7 +32,10 @@ public sealed record UserSettings( bool SshStrictHostKey = true, int SshDefaultPort = 22, int SshTerminalFontSize = 14, - string MajesticConfigPath = "/etc/majestic.yaml") + string MajesticConfigPath = "/etc/majestic.yaml", + // Local AI analytics (Phase 15.2). "auto" lets the detector pick the + // platform execution provider with a CPU fallback; "force-cpu" pins CPU. + string AiAcceleration = "auto") { public static UserSettings Default => new(); } diff --git a/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs b/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs index 7bcfeba..66cacf8 100644 --- a/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs +++ b/src/OpenIPC.Viewer.App/Services/UserSettingsService.cs @@ -28,6 +28,10 @@ public sealed class UserSettingsService : IUserSettingsAccessor public int SshDefaultPort => Current.SshDefaultPort < 1 ? 22 : Current.SshDefaultPort; public string MajesticConfigPath => string.IsNullOrWhiteSpace(Current.MajesticConfigPath) ? "/etc/majestic.yaml" : Current.MajesticConfigPath; + public OpenIPC.Viewer.Core.Analytics.AiAcceleration AiAcceleration => + string.Equals(Current.AiAcceleration, "force-cpu", StringComparison.OrdinalIgnoreCase) + ? OpenIPC.Viewer.Core.Analytics.AiAcceleration.ForceCpu + : OpenIPC.Viewer.Core.Analytics.AiAcceleration.Auto; private static readonly JsonSerializerOptions JsonOpts = new() { diff --git a/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs new file mode 100644 index 0000000..ec415d7 --- /dev/null +++ b/src/OpenIPC.Viewer.App/ViewModels/AnalyticsPageViewModel.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Avalonia.Threading; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using Microsoft.Extensions.Logging; +using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; +using OpenIPC.Viewer.Core.Events; +using OpenIPC.Viewer.Core.Services; + +namespace OpenIPC.Viewer.App.ViewModels; + +// AI control center (Phase 15.7): engine status + diagnostics, the cameras with +// analytics on, and recent detection events. The diagnostics counters poll on a +// 1 Hz timer that the view starts/stops with its visual lifetime. +public sealed partial class AnalyticsPageViewModel : ViewModelBase +{ + private readonly IAnalyticsEngine _engine; + private readonly CameraDirectoryService _directory; + private readonly IEventRepository _events; + private readonly ILogger _logger; + private readonly DispatcherTimer _timer; + + public string Title => Localizer.Instance["Nav.Analytics"]; + + [ObservableProperty] private bool _engineReady; + [ObservableProperty] private string _engineStatus = ""; + [ObservableProperty] private string _activeProvider = "—"; + [ObservableProperty] private long _framesProcessed; + [ObservableProperty] private long _framesDropped; + [ObservableProperty] private int _queueDepth; + [ObservableProperty] private double _averageLatencyMs; + [ObservableProperty] private int _activeCameras; + + public ObservableCollection Cameras { get; } = new(); + public ObservableCollection RecentDetections { get; } = new(); + + public AnalyticsPageViewModel( + IAnalyticsEngine engine, + CameraDirectoryService directory, + IEventRepository events, + ILogger logger) + { + _engine = engine; + _directory = directory; + _events = events; + _logger = logger; + _timer = new DispatcherTimer { Interval = TimeSpan.FromSeconds(1) }; + _timer.Tick += (_, _) => RefreshDiagnostics(); + } + + // Called by the view when shown. + public async Task StartAsync() + { + await RefreshAsync(CancellationToken.None).ConfigureAwait(true); + _timer.Start(); + } + + // Called by the view when hidden — stop polling. + public void Stop() => _timer.Stop(); + + [RelayCommand] + private Task RefreshAsync(CancellationToken ct) => ReloadAsync(ct); + + private async Task ReloadAsync(CancellationToken ct) + { + RefreshDiagnostics(); + try + { + var cameras = await _directory.ListAsync(ct).ConfigureAwait(true); + var names = cameras.ToDictionary(c => c.Id, c => c.Name); + + Cameras.Clear(); + foreach (var c in cameras.Where(c => c.AnalyticsOrDefault.Enabled)) + { + var s = c.AnalyticsOrDefault; + Cameras.Add(new AnalyticsCameraRow( + c.Name, ClassesSummary(s.ClassIds), s.AnalyticsFps, s.ConfidenceThreshold, s.AutoRecord)); + } + + var events = await _events.ListAsync(null, EventKind.Detection, null, 50, ct).ConfigureAwait(true); + RecentDetections.Clear(); + foreach (var ev in events) + { + var name = names.TryGetValue(ev.CameraId, out var n) ? n : ev.CameraId.ToString(); + RecentDetections.Add(new DetectionEventRow( + name, ev.OccurredAt.ToLocalTime(), ev.Summary ?? "")); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load analytics control center data."); + } + } + + private void RefreshDiagnostics() + { + var d = _engine.Diagnostics; + EngineReady = _engine.IsReady; + EngineStatus = StatusLabel(_engine.Status); + ActiveProvider = _engine.ActiveProvider.ToString(); + FramesProcessed = d.FramesProcessed; + FramesDropped = d.FramesDropped; + QueueDepth = d.QueueDepth; + AverageLatencyMs = d.AverageLatencyMs; + ActiveCameras = d.ActiveCameras; + } + + private static string StatusLabel(AnalyticsEngineStatus status) => status switch + { + AnalyticsEngineStatus.Preparing => Localizer.Instance["Analytics.Status.Preparing"], + AnalyticsEngineStatus.Loading => Localizer.Instance["Analytics.Status.Loading"], + AnalyticsEngineStatus.Ready => Localizer.Instance["Analytics.Status.Ready"], + AnalyticsEngineStatus.Failed => Localizer.Instance["Analytics.Status.Failed"], + _ => Localizer.Instance["Analytics.Status.NotStarted"], + }; + + private static string ClassesSummary(IReadOnlyCollection? ids) + { + if (ids is not { Count: > 0 }) return Localizer.Instance["Analytics.AllClasses"]; + return string.Join(", ", ids.Select(id => + id >= 0 && id < CocoClasses.Names.Count ? CocoClasses.Names[id] : $"class{id}")); + } +} + +public sealed record AnalyticsCameraRow( + string Name, string Classes, int Fps, float Threshold, bool AutoRecord); + +public sealed record DetectionEventRow(string CameraName, DateTime OccurredAt, string Summary); diff --git a/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs index 5d9e01d..82068ee 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/CameraTileViewModel.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Reactive.Linq; using System.Threading; using System.Threading.Tasks; @@ -9,7 +10,9 @@ using Microsoft.Extensions.Logging; using OpenIPC.Viewer.App.Messages; using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; +using OpenIPC.Viewer.Core.Events; using OpenIPC.Viewer.Core.Services; using OpenIPC.Viewer.Core.Snapshots; using OpenIPC.Viewer.Core.Video; @@ -22,6 +25,8 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl private readonly CameraDirectoryService _directory; private readonly UserSettingsService _userSettings; private readonly ISnapshotService _snapshots; + private readonly IAnalyticsEngine _analytics; + private readonly AnalyticsBootstrap _analyticsBootstrap; private readonly ILogger _logger; // Auto SD/HD (Phase 12.2): substream in the grid, mainstream when a single @@ -31,8 +36,10 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl private StreamQuality _quality = StreamQuality.Sub; private IDisposable? _stateSub; private IDisposable? _telemetrySub; + private IDisposable? _resultsSub; private bool _started; private bool _disposed; + private bool _suspended; public Camera Camera { get; } @@ -54,6 +61,20 @@ public sealed partial class CameraTileViewModel : ViewModelBase, IAsyncDisposabl [NotifyPropertyChangedFor(nameof(ErrorDetail))] private string? _errorMessage; + // Latest detections for this tile (Phase 15.5), normalized 0..1 boxes drawn + // by the DetectionOverlay control. Updated off the inference worker thread. + [ObservableProperty] + [NotifyPropertyChangedFor(nameof(DetectionCounter))] + [NotifyPropertyChangedFor(nameof(HasDetections))] + private IReadOnlyList _detections = Array.Empty(); + + public bool AnalyticsEnabled => Camera.AnalyticsOrDefault.Enabled; + + // Bottom-center counter badge, e.g. "person ×2, car ×1". + public string DetectionCounter => AnalyticsMotionEventSource.Summarize(Detections); + + public bool HasDetections => AnalyticsEnabled && Detections.Count > 0; + public string Name => Camera.Name; public string StateLabel => State switch { @@ -116,6 +137,8 @@ public CameraTileViewModel( CameraDirectoryService directory, UserSettingsService userSettings, ISnapshotService snapshots, + IAnalyticsEngine analytics, + AnalyticsBootstrap analyticsBootstrap, ILogger logger) { Camera = camera; @@ -123,9 +146,17 @@ public CameraTileViewModel( _directory = directory; _userSettings = userSettings; _snapshots = snapshots; + _analytics = analytics; + _analyticsBootstrap = analyticsBootstrap; _logger = logger; _coordinator.Invalidated += OnCoordinatorInvalidated; + + // Always listen for results for this camera; they only arrive while the + // tile is attached + the engine is ready, so this is cheap otherwise. + _resultsSub = _analytics.Results + .Where(r => r.CameraId == Camera.Id) + .Subscribe(r => Dispatcher.UIThread.Post(() => Detections = r.Detections)); } // Set the stream quality before the first ActivateAsync. No-op once started @@ -145,6 +176,7 @@ public async Task SetQualityAsync(StreamQuality quality, CancellationToken ct) if (_disposed || quality == _quality) return; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; @@ -188,6 +220,8 @@ public async Task ActivateAsync(CancellationToken ct) if (session.State == SessionState.Idle) await session.StartAsync(ct).ConfigureAwait(true); + + AttachAnalytics(session); } catch (Exception ex) { @@ -196,6 +230,27 @@ public async Task ActivateAsync(CancellationToken ct) } } + // Tap this session's frames for object detection (Phase 15.3) when the + // camera has analytics on. Kicks the engine bootstrap (model load) in the + // background; frames are dropped until it's ready. isActive gates analytics + // off for a Smart-Paused / non-playing tile. + private void AttachAnalytics(IVideoSession session) + { + if (!Camera.AnalyticsOrDefault.Enabled) return; + _ = _analyticsBootstrap.EnsureStartedAsync(); + _analytics.Attach( + Camera.Id, + session.Frames, + () => Camera.AnalyticsOrDefault, + () => !_suspended && State == SessionState.Playing); + } + + private void DetachAnalytics() + { + _analytics.Detach(Camera.Id); + Detections = Array.Empty(); + } + // Coordinator dropped every cached session (e.g. RtspTransport flipped in // Settings). Our current Session ref is now disposed — drop subscriptions, // reset state, and re-Acquire on the UI thread so observable updates land @@ -209,6 +264,7 @@ private void OnCoordinatorInvalidated(object? sender, EventArgs e) { _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); Session = null; State = SessionState.Idle; _started = false; @@ -261,6 +317,7 @@ private async Task RetryAsync() ErrorMessage = null; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; @@ -278,9 +335,20 @@ private void Close() => WeakReferenceMessenger.Default.Send(new CloseTileMessage(Camera.Id)); // Smart Pause (Phase 12.1): suspend/resume decode without dropping the - // session, so the last frame stays frozen for an instant resume. - public void Pause() => Session?.PauseDecode(); - public void Resume() => Session?.Resume(); + // session, so the last frame stays frozen for an instant resume. Analytics + // pauses with the tile (12.1 ↔ 15.3) — isActive reads _suspended. + public void Pause() + { + _suspended = true; + Session?.PauseDecode(); + Detections = Array.Empty(); + } + + public void Resume() + { + _suspended = false; + Session?.Resume(); + } public async ValueTask DisposeAsync() { @@ -288,6 +356,8 @@ public async ValueTask DisposeAsync() _coordinator.Invalidated -= OnCoordinatorInvalidated; _stateSub?.Dispose(); _telemetrySub?.Dispose(); + _resultsSub?.Dispose(); + DetachAnalytics(); if (Session is not null) { Session = null; diff --git a/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs index c63352b..5403ce2 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/Dialogs/CameraEditorViewModel.cs @@ -10,6 +10,7 @@ using CommunityToolkit.Mvvm.Input; using Microsoft.Extensions.Logging; using OpenIPC.Viewer.App.Services; +using OpenIPC.Viewer.Core.Analytics; using OpenIPC.Viewer.Core.Entities; using OpenIPC.Viewer.Core.Services; using OpenIPC.Viewer.Core.Video; @@ -53,6 +54,16 @@ public sealed partial class CameraEditorViewModel : ViewModelBase [ObservableProperty] private StreamQualityOption? _selectedStreamQuality; + // AI analytics (Phase 15.4). Threshold 0..1; fps clamped 1..10 by the engine. + [ObservableProperty] private bool _aiEnabled; + [ObservableProperty] private double _aiThreshold = 0.5; + [ObservableProperty] private int _aiFps = 3; + [ObservableProperty] private bool _aiAutoRecord; + [ObservableProperty] private int _aiPostEventSeconds = 15; + + // Curated, surveillance-relevant subset of COCO classes the user can toggle. + public ObservableCollection AnalyticsClasses { get; } = new(); + // Includes a leading null entry so the user can pick "no group". public ObservableCollection AvailableGroups { get; } = new(); @@ -74,6 +85,21 @@ public CameraEditorViewModel(IVideoEngine engine, CameraDirectoryService directo _userSettings = userSettings; _logger = logger; SelectedStreamQuality = StreamQualityOptions[0]; // Auto + PopulateAnalyticsClasses(); + } + + // Surveillance-relevant COCO classes. person is pre-selected so a fresh + // enable does something sensible; the user can broaden/narrow from here. + private static readonly (int Id, bool Default)[] CuratedClasses = + { + (0, true), (1, false), (2, false), (3, false), (5, false), (7, false), (15, false), (16, false), + }; + + private void PopulateAnalyticsClasses() + { + if (AnalyticsClasses.Count > 0) return; + foreach (var (id, def) in CuratedClasses) + AnalyticsClasses.Add(new DetectionClassOption(id, CocoClasses.Names[id]) { IsSelected = def }); } public CameraEditorViewModel(Camera existing, CameraCredentials? credentials, CameraCredentials? sshCredentials, IVideoEngine engine, CameraDirectoryService directory, UserSettingsService userSettings, ILogger logger) @@ -94,6 +120,19 @@ public CameraEditorViewModel(Camera existing, CameraCredentials? credentials, Ca _pendingGroupId = existing.GroupId; SelectedStreamQuality = StreamQualityOptions.FirstOrDefault(o => o.Value == existing.StreamQualityOverride) ?? StreamQualityOptions[0]; + + var ai = existing.AnalyticsOrDefault; + AiEnabled = ai.Enabled; + AiThreshold = ai.ConfidenceThreshold; + AiFps = ai.AnalyticsFps; + AiAutoRecord = ai.AutoRecord; + AiPostEventSeconds = ai.PostEventSeconds; + if (ai.ClassIds is { Count: > 0 } ids) + { + var set = new HashSet(ids); + foreach (var opt in AnalyticsClasses) + opt.IsSelected = set.Contains(opt.ClassId); + } } public async Task LoadGroupsAsync(CancellationToken ct) @@ -187,6 +226,7 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe : new CameraCredentials(SshUsername, SshPassword); var quality = SelectedStreamQuality?.Value ?? StreamQualityOverride.Auto; + var analytics = BuildAnalyticsSettings(); if (EditingId is null) { @@ -201,7 +241,8 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe GroupId: SelectedGroup?.Id, StreamQualityOverride: quality, SshCredentials: sshCredentials, - SshPort: sshPort); + SshPort: sshPort, + Analytics: analytics); } else { @@ -216,7 +257,8 @@ public bool TryBuildRequest(out NewCameraRequest? newRequest, out UpdateCameraRe GroupId: SelectedGroup?.Id, StreamQualityOverride: quality, SshCredentials: sshCredentials, - SshPort: sshPort); + SshPort: sshPort, + Analytics: analytics); } return ok; @@ -298,6 +340,18 @@ private bool TryValidate(out bool ok, out Uri rtspMain, out Uri? rtspSub, out in return true; } + private AnalyticsSettings BuildAnalyticsSettings() + { + var classIds = AnalyticsClasses.Where(c => c.IsSelected).Select(c => c.ClassId).ToArray(); + return new AnalyticsSettings( + Enabled: AiEnabled, + ClassIds: classIds.Length > 0 ? classIds : null, + ConfidenceThreshold: (float)AiThreshold, + AnalyticsFps: AiFps, + AutoRecord: AiAutoRecord, + PostEventSeconds: AiPostEventSeconds); + } + private static RtspTransport ParseTransport(string? s) => s?.ToLowerInvariant() switch { "udp" => RtspTransport.Udp, @@ -305,6 +359,21 @@ private bool TryValidate(out bool ok, out Uri rtspMain, out Uri? rtspSub, out in }; } +// Toggleable detection class in the editor (Phase 15.4). +public sealed partial class DetectionClassOption : ObservableObject +{ + public DetectionClassOption(int classId, string name) + { + ClassId = classId; + Name = name; + } + + public int ClassId { get; } + public string Name { get; } + + [ObservableProperty] private bool _isSelected; +} + public sealed record CameraEditorResult(NewCameraRequest? NewRequest, UpdateCameraRequest? UpdateRequest); // Combo item for the per-camera SD/HD override picker (Phase 12.2). diff --git a/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs index b3fcfd2..94ab930 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/GridPageViewModel.cs @@ -27,6 +27,8 @@ public sealed partial class GridPageViewModel : ViewModelBase, private readonly LiveStreamCoordinator _coordinator; private readonly UserSettingsService _userSettings; private readonly ISnapshotService _snapshots; + private readonly OpenIPC.Viewer.Core.Analytics.IAnalyticsEngine _analytics; + private readonly AnalyticsBootstrap _analyticsBootstrap; private readonly ILoggerFactory _loggerFactory; private readonly ILogger _logger; @@ -52,12 +54,16 @@ public GridPageViewModel( LiveStreamCoordinator coordinator, UserSettingsService userSettings, ISnapshotService snapshots, + OpenIPC.Viewer.Core.Analytics.IAnalyticsEngine analytics, + AnalyticsBootstrap analyticsBootstrap, ILoggerFactory loggerFactory) { _directory = directory; _coordinator = coordinator; _userSettings = userSettings; _snapshots = snapshots; + _analytics = analytics; + _analyticsBootstrap = analyticsBootstrap; _loggerFactory = loggerFactory; _logger = loggerFactory.CreateLogger(); @@ -126,7 +132,10 @@ private async Task RefreshTilesAsync(CancellationToken ct) var existing = Tiles.FirstOrDefault(t => t.Camera.Id == camera.Id); if (existing is not null) { - if (!StreamUriChanged(existing.Camera, camera)) + // Rebuild on a stream-URL change OR an analytics-settings change: + // the tile holds an immutable Camera snapshot, so toggling + // analytics in the editor only takes effect by rebuilding it. + if (!StreamUriChanged(existing.Camera, camera) && !AnalyticsChanged(existing.Camera, camera)) { // Kept tile — re-evaluate SD/HD against the (possibly new) // layout. No-op when quality is unchanged. @@ -140,7 +149,7 @@ private async Task RefreshTilesAsync(CancellationToken ct) try { await existing.DisposeAsync().ConfigureAwait(true); } catch (Exception ex) { _logger.LogWarning(ex, "Error releasing stale tile for {Camera}", camera.Name); } - var rebuilt = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _loggerFactory.CreateLogger()); + var rebuilt = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _analytics, _analyticsBootstrap, _loggerFactory.CreateLogger()); rebuilt.SetInitialQuality(quality); Tiles.Insert(idx, rebuilt); try { await rebuilt.ActivateAsync(ct).ConfigureAwait(true); } @@ -148,7 +157,7 @@ private async Task RefreshTilesAsync(CancellationToken ct) continue; } - var tile = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _loggerFactory.CreateLogger()); + var tile = new CameraTileViewModel(camera, _coordinator, _directory, _userSettings, _snapshots, _analytics, _analyticsBootstrap, _loggerFactory.CreateLogger()); tile.SetInitialQuality(quality); Tiles.Add(tile); try { await tile.ActivateAsync(ct).ConfigureAwait(true); } @@ -170,6 +179,23 @@ private async Task RefreshTilesAsync(CancellationToken ct) private static bool StreamUriChanged(Camera a, Camera b) => (a.RtspSubUri ?? a.RtspMainUri) != (b.RtspSubUri ?? b.RtspMainUri); + // True when a camera's analytics config changed (Phase 15) — the tile's + // frame tap reads its immutable Camera snapshot, so any change needs a + // rebuild. Compared field-by-field because AnalyticsSettings.ClassIds is a + // collection (record equality would compare it by reference). + private static bool AnalyticsChanged(Camera a, Camera b) + { + var x = a.AnalyticsOrDefault; + var y = b.AnalyticsOrDefault; + if (x.Enabled != y.Enabled || x.AutoRecord != y.AutoRecord || x.AnalyticsFps != y.AnalyticsFps + || x.PostEventSeconds != y.PostEventSeconds + || Math.Abs(x.ConfidenceThreshold - y.ConfidenceThreshold) > 0.001f) + return true; + var xc = (x.ClassIds ?? Array.Empty()).OrderBy(i => i); + var yc = (y.ClassIds ?? Array.Empty()).OrderBy(i => i); + return !xc.SequenceEqual(yc); + } + // Auto SD/HD policy (Phase 12.2): mainstream only when a single tile fills // the view (1×1 layout) and the user hasn't disabled the feature; otherwise // the substream keeps the multi-camera grid light. diff --git a/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs index a50983f..d1d93ec 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/MainWindowViewModel.cs @@ -23,6 +23,7 @@ public sealed partial class MainWindowViewModel : ViewModelBase, IRecipient CurrentPage is CameraLibraryPageViewModel or SingleCameraPageViewModel; public bool IsRecordingsSelected => CurrentPage is RecordingsPageViewModel; public bool IsEventsSelected => CurrentPage is EventsPageViewModel; + public bool IsAnalyticsSelected => CurrentPage is AnalyticsPageViewModel; public bool IsSettingsSelected => CurrentPage is SettingsPageViewModel; public MainWindowViewModel( @@ -52,6 +55,7 @@ public MainWindowViewModel( CameraLibraryPageViewModel library, RecordingsPageViewModel recordings, EventsPageViewModel events, + AnalyticsPageViewModel analytics, SettingsPageViewModel settings, CameraDirectoryService directory, SingleCameraPageFactory singleCameraFactory, @@ -61,6 +65,7 @@ public MainWindowViewModel( Library = library; Recordings = recordings; Events = events; + Analytics = analytics; Settings = settings; _directory = directory; _singleCameraFactory = singleCameraFactory; @@ -113,6 +118,7 @@ private void Navigate(string target) "library" => Library, "recordings" => Recordings, "events" => Events, + "analytics" => Analytics, "settings" => Settings, _ => CurrentPage, }; diff --git a/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs b/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs index bdff6dc..b6be0cf 100644 --- a/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs +++ b/src/OpenIPC.Viewer.App/ViewModels/SettingsPageViewModel.cs @@ -31,6 +31,9 @@ public sealed partial class SettingsPageViewModel : ViewModelBase [ObservableProperty] private string _rtspTransport = "tcp"; [ObservableProperty] private bool _autoSdHd = true; + // AI analytics (Phase 15.2). On → pin the CPU execution provider. + [ObservableProperty] private bool _aiForceCpu; + [ObservableProperty] private NetworkInterfaceOption? _selectedNetworkInterface; [ObservableProperty] private string _language = "system"; @@ -150,6 +153,7 @@ private void Load() MaxConcurrentGridSessions = s.MaxConcurrentGridSessions; RtspTransport = s.RtspTransport; AutoSdHd = s.AutoSdHd; + AiForceCpu = string.Equals(s.AiAcceleration, "force-cpu", StringComparison.OrdinalIgnoreCase); SelectedNetworkInterface = NetworkInterfaceOptions.FirstOrDefault(o => o.Value == s.PreferredNetworkInterface) ?? NetworkInterfaceOptions[0]; @@ -170,6 +174,7 @@ private void Load() partial void OnMaxConcurrentGridSessionsChanged(int value) => Persist(); partial void OnRtspTransportChanged(string value) => Persist(); partial void OnAutoSdHdChanged(bool value) => Persist(); + partial void OnAiForceCpuChanged(bool value) => Persist(); partial void OnSelectedNetworkInterfaceChanged(NetworkInterfaceOption? value) => Persist(); partial void OnRecordingsDirOverrideChanged(string value) => Persist(); partial void OnLanguageChanged(string value) => Persist(); @@ -190,6 +195,7 @@ private void Persist() MaxConcurrentGridSessions = MaxConcurrentGridSessions, RtspTransport = RtspTransport, AutoSdHd = AutoSdHd, + AiAcceleration = AiForceCpu ? "force-cpu" : "auto", PreferredNetworkInterface = SelectedNetworkInterface?.Value ?? "", RecordingsDirOverride = RecordingsDirOverride, Language = Language, diff --git a/src/OpenIPC.Viewer.App/Views/BottomNavBar.axaml b/src/OpenIPC.Viewer.App/Views/BottomNavBar.axaml index 8422235..ad5c73b 100644 --- a/src/OpenIPC.Viewer.App/Views/BottomNavBar.axaml +++ b/src/OpenIPC.Viewer.App/Views/BottomNavBar.axaml @@ -10,7 +10,7 @@ BorderThickness="0,1,0,0" Padding="8,4"> - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +