diff --git a/.machine_readable/6a2/NEUROSYM.a2ml b/.machine_readable/6a2/NEUROSYM.a2ml index 9a0144f..639d4ef 100644 --- a/.machine_readable/6a2/NEUROSYM.a2ml +++ b/.machine_readable/6a2/NEUROSYM.a2ml @@ -22,7 +22,7 @@ rules = [ { name = "no-opus-pretence", pattern = "opus_encode|opus_decode", severity = "warning", scope = "*.ex", note = "Backend.audio_encode is PCM framing, not Opus. Use opus_transcode/4 explicitly." }, { name = "stub-nif-returns-error", pattern = "simulated response", severity = "critical", scope = "*.ex", note = "Burble.LLM.process_query must not return simulated strings in production" }, { name = "no-system-time-outside-ptp", pattern = "System\\.system_time", severity = "warning", scope = "*.ex", note = "Should go through Burble.Timing.PTP.now/0 for clock-source awareness" }, - { name = "tflite-model-path-validated", pattern = "nif_neural_init_model", severity = "warning", scope = "*.ex", note = "Model path not validated; model file not in priv/" }, + { name = "neural-is-spectral-gating", pattern = "nif_neural_init_model", severity = "info", scope = "*.ex", note = "Neural denoiser is Phase 1 spectral gating (ffi/zig/src/coprocessor/neural.zig), not TFLite. Works as-is. Phase 2 (RNNoise) is planned but not blocking." }, ] [neural-config] diff --git a/.machine_readable/6a2/STATE.a2ml b/.machine_readable/6a2/STATE.a2ml index 431f90e..a6d9b39 100644 --- a/.machine_readable/6a2/STATE.a2ml +++ b/.machine_readable/6a2/STATE.a2ml @@ -24,7 +24,7 @@ milestones = [ { name = "v0.1.0 to v0.4.0 — Foundation & Transport", completion = 100 }, { name = "v1.0.0 — Stable Release", completion = 100 }, { name = "Phase 0 — Scrub baseline (V-lang removed, docs honest)", completion = 100, date = "2026-04-16" }, - { name = "Phase 1 — Audio dependable (Opus honest, jitter sync, comfort noise, REMB, Avow chain)", completion = 0 }, + { name = "Phase 1 — Audio dependable (Opus honest, comfort noise, REMB, Avow chain, echo-cancel ref, neural spectral-gate verified)", completion = 85 }, { name = "Phase 2 — P2P AI channel dependable (burble-ai-bridge fixes, round-trip tests, docs) — CRITICAL PATH for family/pair-programming use case", completion = 30 }, { name = "Phase 2b — server-side Burble.LLM (provider, circuit breaker, fixed parse_frame, NimblePool wired) — SECONDARY, not required for family use case", completion = 0 }, { name = "Phase 3 — RTSP + signaling + text + AffineScript client start", completion = 0 }, @@ -61,11 +61,12 @@ phase-2-p2p-ai-bridge = [ ] phase-1-audio = [ "DONE 2026-04-16: Opus honest contract (opus_transcode returns :not_implemented)", - "NEXT: Validate TFLite neural model or gate behind feature flag", - "NEXT: Wire RTP-timestamp jitter sync across peers (precursor to PTP phase)", - "NEXT: Server-side comfort noise injection on RX silence", - "NEXT: REMB bitrate adaptation feedback loop", - "NEXT: Replace Avow stub with hash-chain audit log + non-circularity property test" + "DONE 2026-04-16: Neural denoiser is spectral gating (not TFLite) — already working, no gating needed. TFLite concern was about deleted api/zig/.", + "DONE 2026-04-16: Pipeline echo cancel now uses real playback reference (was hardcoded silence — always no-op)", + "DONE 2026-04-16: Server-side comfort noise injection after 3 silent frames (60ms at 20ms/frame)", + "DONE 2026-04-16: REMB bitrate adaptation — Pipeline.update_bitrate/3 wired via Backend.io_adaptive_bitrate", + "DONE 2026-04-16: Avow hash-chain linkage + ETS store + 10 property tests (commit 43669aa)", + "NEXT: Wire RTP-timestamp sync in media/peer.ex → Pipeline (precursor to PTP Phase 4, not blocking audio)" ] [maintenance-status] diff --git a/server/lib/burble/coprocessor/pipeline.ex b/server/lib/burble/coprocessor/pipeline.ex index 7dce498..e6dbda0 100644 --- a/server/lib/burble/coprocessor/pipeline.ex +++ b/server/lib/burble/coprocessor/pipeline.ex @@ -132,6 +132,14 @@ defmodule Burble.Coprocessor.Pipeline do neural_state: neural_state, jitter_buffer: %{}, prev_frames: [], + # Playback reference for echo cancellation — populated from decoded + # inbound frames so the echo canceller has a real speaker signal to + # subtract from the capture. When this is empty (no inbound audio yet), + # echo cancel runs against silence (harmless no-op until first frame). + playback_ref: [], + # Silence counter: frames since last non-nil inbound. Drives comfort + # noise injection so peers don't hear dead air when a speaker pauses. + silence_frames: 0, # Metrics frames_processed: 0, frames_dropped: 0, @@ -158,9 +166,13 @@ defmodule Burble.Coprocessor.Pipeline do # Step 2: Noise gate. pcm = Backend.audio_noise_gate(pcm, config.noise_gate_db) - # Step 3: Echo cancellation (needs reference — use silence if none). - # In production, the reference comes from the playback buffer. - reference = List.duplicate(0.0, length(pcm)) + # Step 3: Echo cancellation — use real playback reference when available. + reference = + case state.playback_ref do + ref when is_list(ref) and length(ref) == length(pcm) -> ref + _ -> List.duplicate(0.0, length(pcm)) + end + pcm = Backend.audio_echo_cancel(pcm, reference, config.echo_cancel_taps) # Step 4: Encode. @@ -202,8 +214,18 @@ defmodule Burble.Coprocessor.Pipeline do case buffered do nil -> - # Buffer not ready to emit — need more packets. - {:reply, {:ok, nil}, state} + # Buffer not ready to emit — need more packets. If we've been + # silent for enough frames, inject comfort noise so the peer + # doesn't hear dead air. This is a server-side injection only; + # the client's own comfort noise generator handles the local side. + silence_frames = state.silence_frames + 1 + + if silence_frames >= 3 do + comfort = Backend.audio_comfort_noise(960, -60.0, %{}) + {:reply, {:ok, comfort}, %{state | silence_frames: silence_frames}} + else + {:reply, {:ok, nil}, %{state | silence_frames: silence_frames}} + end ready_frame -> # Step 2: Check for loss (gap in sequence numbers handled by jitter buffer). @@ -230,7 +252,9 @@ defmodule Burble.Coprocessor.Pipeline do {:ok, pcm} -> new_state = %{state | frames_processed: state.frames_processed + 1, - prev_frames: [ready_frame | Enum.take(state.prev_frames, 2)] + prev_frames: [ready_frame | Enum.take(state.prev_frames, 2)], + playback_ref: pcm, + silence_frames: 0 } {:reply, {:ok, pcm}, new_state} @@ -273,6 +297,36 @@ defmodule Burble.Coprocessor.Pipeline do {:reply, {:ok, health}, state} end + # --------------------------------------------------------------------------- + # Bitrate adaptation (REMB feedback) + # --------------------------------------------------------------------------- + + @doc """ + Update the encoding bitrate based on REMB (Receiver Estimated Maximum + Bitrate) feedback from the peer's PeerConnection. + + Called by `Burble.Media.Peer` when it receives an RTCP REMB packet + indicating the remote client's available bandwidth. The pipeline adjusts + its PCM framing bitrate accordingly (primarily affects self-test and + archive paths; live SFU forwarding is opaque Opus which the browser + adjusts independently). + """ + def update_bitrate(pipeline, loss_ratio, rtt_ms) do + GenServer.cast(pipeline, {:update_bitrate, loss_ratio, rtt_ms}) + end + + @impl true + def handle_cast({:update_bitrate, loss_ratio, rtt_ms}, state) do + new_bitrate = Backend.io_adaptive_bitrate(loss_ratio, rtt_ms, state.current_bitrate) + + if new_bitrate != state.current_bitrate do + Logger.info("[Pipeline] Bitrate #{state.current_bitrate} → #{new_bitrate} " <> + "(loss=#{Float.round(loss_ratio * 100, 1)}%, rtt=#{rtt_ms}ms)") + end + + {:noreply, %{state | current_bitrate: new_bitrate}} + end + # --------------------------------------------------------------------------- # Private helpers # ---------------------------------------------------------------------------