From 1b7f27db53a2e39af8c79e9d7a83c216313c2c6d Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Mon, 6 Apr 2026 15:56:02 +0200 Subject: [PATCH 1/9] Initial set of changes for gst pipeline to support non live sources --- pkg/config/base.go | 2 +- pkg/config/pipeline.go | 6 +- pkg/gstreamer/race_test.go | 459 ++++++++++++++++++ pkg/handler/handler.go | 2 +- pkg/pipeline/builder/audio.go | 136 +++++- pkg/pipeline/builder/file.go | 5 + pkg/pipeline/builder/video.go | 21 +- pkg/pipeline/controller.go | 102 ++-- pkg/pipeline/source/testfeeder/feeder.go | 235 +++++++++ pkg/pipeline/source/testfeeder/reader_h264.go | 81 ++++ pkg/pipeline/source/testfeeder/reader_ivf.go | 93 ++++ pkg/pipeline/source/testfeeder/reader_ogg.go | 148 ++++++ pkg/pipeline/source/testfeeder/source.go | 272 +++++++++++ pkg/pipeline/source/testfeeder/source_test.go | 153 ++++++ pkg/pipeline/watch.go | 8 +- 15 files changed, 1676 insertions(+), 47 deletions(-) create mode 100644 pkg/gstreamer/race_test.go create mode 100644 pkg/pipeline/source/testfeeder/feeder.go create mode 100644 pkg/pipeline/source/testfeeder/reader_h264.go create mode 100644 pkg/pipeline/source/testfeeder/reader_ivf.go create mode 100644 pkg/pipeline/source/testfeeder/reader_ogg.go create mode 100644 pkg/pipeline/source/testfeeder/source.go create mode 100644 pkg/pipeline/source/testfeeder/source_test.go diff --git a/pkg/config/base.go b/pkg/config/base.go index 4ebfb8ba1..7f7b654a0 100644 --- a/pkg/config/base.go +++ b/pkg/config/base.go @@ -59,7 +59,7 @@ type BaseConfig struct { S3AssumeRoleExternalID string `yaml:"s3_assume_role_external_id"` // if set, this external ID is used by default for S3 uploads // advanced - Insecure bool `yaml:"insecure"` // allow chrome to connect to an insecure websocket + Insecure bool `yaml:"insecure"` // allow chrome to connect to an insecure websocket, bypasses chrome LNA checks Debug DebugConfig `yaml:"debug"` // create dot file on internal error ChromeFlags map[string]interface{} `yaml:"chrome_flags"` // additional flags to pass to Chrome Latency LatencyConfig `yaml:"latency"` // gstreamer latencies, modifying these may break the service diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index 71a0e21bd..1d007d561 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -59,7 +59,7 @@ type PipelineConfig struct { Info *livekit.EgressInfo `yaml:"-"` Manifest *Manifest `yaml:"-"` StorageReporter storageobs.ProjectReporter `yaml:"-"` - IsReplay bool `yaml:"-"` + Live bool `yaml:"-"` } var ( @@ -151,6 +151,7 @@ func NewPipelineConfig(confString string, req *rpc.StartEgressRequest) (*Pipelin }, Outputs: make(map[types.EgressType][]OutputConfig), StorageReporter: storageobs.NewNoopProjectReporter(), + Live: true, } if err := yaml.Unmarshal([]byte(confString), p); err != nil { @@ -177,6 +178,7 @@ func GetValidatedPipelineConfig(conf *ServiceConfig, req *rpc.StartEgressRequest BaseConfig: conf.BaseConfig, TmpDir: path.Join(TmpDir, req.EgressId), Outputs: make(map[types.EgressType][]OutputConfig), + Live: true, } return p, p.Update(req) @@ -417,7 +419,7 @@ func (p *PipelineConfig) Update(request *rpc.StartEgressRequest) error { } case *rpc.StartEgressRequest_Replay: - p.IsReplay = true + p.Live = false replayReq := req.Replay clone := proto.Clone(replayReq).(*livekit.ExportReplayRequest) p.Info.Request = &livekit.EgressInfo_Replay{ diff --git a/pkg/gstreamer/race_test.go b/pkg/gstreamer/race_test.go new file mode 100644 index 000000000..e8a1f3733 --- /dev/null +++ b/pkg/gstreamer/race_test.go @@ -0,0 +1,459 @@ +package gstreamer + +import ( + "fmt" + "math/rand" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/go-gst/go-glib/glib" + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/stretchr/testify/require" +) + +var gstInitOnce sync.Once + +func initGStreamer(t *testing.T) { + t.Helper() + gstInitOnce.Do(func() { + gst.Init(nil) + }) +} + +// TestAddSourceBinRace attempts to reproduce the race condition where a source +// bin is added via AddSourceBin concurrently with the pipeline's PAUSED → PLAYING +// transition. In production this causes either FlowFlushing (priv->flushing stuck +// true) or a dead streaming thread (need-data never fires). +// +// Pipeline topology matches production: +// +// Source bins: appsrc → rtpopusdepay → opusdec → audiorate → queue(leaky) → audioconvert → audioresample → capsfilter +// Sink bin: audiotestsrc → capsfilter | audiomixer → capsfilter → queue → fakesink +// +// Production-matching timing: +// - GLib main loop running during the race (bus messages processed via watch, same as production) +// - Bus watch triggers concurrent PushBuffer (simulates pushSamples waking on Playing fuse) +// - Timing variants: bin-add-first (production pattern), simultaneous, playing-first +func TestAddSourceBinRace(t *testing.T) { + initGStreamer(t) + + const iterations = 1000 + + var ( + flowFlushing int + needDataMiss int + pushOK int + watchFired int + loopConfirm int + needDataOK int + errCount int + byVariant [3]int // [bin-add-first, simultaneous, playing-first] + ) + + for i := range iterations { + r := runRaceIteration(t, i) + if r.flowFlushing { + flowFlushing++ + } + if r.needDataMissing { + needDataMiss++ + } + if r.pushFlowOK { + pushOK++ + } + if r.watchTriggered { + watchFired++ + } + if r.loopRan { + loopConfirm++ + } + if r.needDataReceived { + needDataOK++ + } + if r.setupError { + errCount++ + } + byVariant[r.variant]++ + } + + t.Logf("results over %d iterations:", iterations) + t.Logf(" race hits: flowFlushing=%d needDataMiss=%d", flowFlushing, needDataMiss) + t.Logf(" healthy: loopConfirmed=%d watchFired=%d pushOK=%d needDataOK=%d", + loopConfirm, watchFired, pushOK, needDataOK) + t.Logf(" errors: setupErrors=%d", errCount) + t.Logf(" variants: binAddFirst=%d simultaneous=%d playingFirst=%d", + byVariant[0], byVariant[1], byVariant[2]) + + if flowFlushing > 0 || needDataMiss > 0 { + t.Logf("RACE DETECTED: reproduced the dynamic bin addition race condition") + } + + // Verify the test machinery actually worked on every non-error iteration + healthy := iterations - errCount + require.Equal(t, healthy, loopConfirm, "GLib main loop should have run on every non-error iteration") + require.Equal(t, healthy, watchFired, "bus watch should have fired on every non-error iteration") + require.Equal(t, healthy, pushOK+flowFlushing, "PushBuffer should have returned OK or FlowFlushing on every non-error iteration") + require.Equal(t, healthy, needDataOK+needDataMiss, "need-data should have been checked on every non-error iteration") +} + +const ( + raceAudioCaps = "audio/x-raw,format=S16LE,rate=48000,channels=2,layout=interleaved" + raceRTPCaps = "application/x-rtp,media=audio,payload=111,encoding-name=OPUS,clock-rate=48000" + // 3ms tolerance matches production audioRateTolerance + raceAudioRateTolerance = 3 * time.Millisecond + // 1s latency for queue max-size-time (production uses PipelineLatency) + raceQueueLatency = time.Second + + variantBinAddFirst = 0 + variantSimultaneous = 1 + variantPlayingFirst = 2 +) + +type raceResult struct { + flowFlushing bool + needDataMissing bool + pushFlowOK bool + watchTriggered bool + loopRan bool + needDataReceived bool + setupError bool + variant int +} + +// buildAudioConverterChain creates the production audio converter chain: +// audiorate → queue(leaky=downstream) → audioconvert → audioresample → capsfilter +func buildAudioConverterChain(t *testing.T) []*gst.Element { + t.Helper() + + audioRate, err := gst.NewElement("audiorate") + require.NoError(t, err) + require.NoError(t, audioRate.SetProperty("skip-to-first", true)) + require.NoError(t, audioRate.SetProperty("tolerance", uint64(raceAudioRateTolerance))) + + audioQueue, err := gst.NewElement("queue") + require.NoError(t, err) + require.NoError(t, audioQueue.SetProperty("max-size-time", uint64(raceQueueLatency))) + require.NoError(t, audioQueue.SetProperty("max-size-bytes", uint(0))) + require.NoError(t, audioQueue.SetProperty("max-size-buffers", uint(0))) + audioQueue.SetArg("leaky", "downstream") + + audioConvert, err := gst.NewElement("audioconvert") + require.NoError(t, err) + + audioResample, err := gst.NewElement("audioresample") + require.NoError(t, err) + + capsFilter, err := gst.NewElement("capsfilter") + require.NoError(t, err) + require.NoError(t, capsFilter.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) + + return []*gst.Element{audioRate, audioQueue, audioConvert, audioResample, capsFilter} +} + +func runRaceIteration(t *testing.T, iteration int) raceResult { + t.Helper() + + var result raceResult + + // Determine timing variant + switch v := iteration % 10; { + case v < 5: + result.variant = variantBinAddFirst + case v < 8: + result.variant = variantSimultaneous + default: + result.variant = variantPlayingFirst + } + + callbacks := &Callbacks{ + GstReady: make(chan struct{}), + BuildReady: make(chan struct{}), + } + close(callbacks.GstReady) + close(callbacks.BuildReady) + + p, err := NewPipeline("test_race", 0, callbacks) + require.NoError(t, err) + + // Build sink bin: audiotestsrc → capsfilter | audiomixer → capsfilter → queue → fakesink + audioBin := p.NewBin("audio") + + audioTestSrc, err := gst.NewElement("audiotestsrc") + require.NoError(t, err) + require.NoError(t, audioTestSrc.SetProperty("volume", 0.0)) + require.NoError(t, audioTestSrc.SetProperty("do-timestamp", true)) + require.NoError(t, audioTestSrc.SetProperty("is-live", true)) + require.NoError(t, audioTestSrc.SetProperty("samplesperbuffer", 960)) // 20ms @ 48kHz + + testSrcCaps, err := gst.NewElement("capsfilter") + require.NoError(t, err) + require.NoError(t, testSrcCaps.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) + + testSrcBin := p.NewBin("audio_test_src") + require.NoError(t, testSrcBin.AddElements(audioTestSrc, testSrcCaps)) + + audioMixer, err := gst.NewElement("audiomixer") + require.NoError(t, err) + + mixerCaps, err := gst.NewElement("capsfilter") + require.NoError(t, err) + require.NoError(t, mixerCaps.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) + + outQueue, err := gst.NewElement("queue") + require.NoError(t, err) + + fakeSink, err := gst.NewElement("fakesink") + require.NoError(t, err) + require.NoError(t, fakeSink.SetProperty("async", false)) + + require.NoError(t, audioBin.AddElements(audioMixer, mixerCaps, outQueue, fakeSink)) + require.NoError(t, audioBin.AddSourceBin(testSrcBin)) + require.NoError(t, p.AddSinkBin(audioBin)) + + // Pre-existing source bins with full production chain to widen race window + const preExistingSources = 5 + preAppSrcs := make([]*app.Source, preExistingSources) + for i := range preExistingSources { + src, err := app.NewAppSrc() + require.NoError(t, err) + src.SetArg("format", "time") + require.NoError(t, src.SetProperty("is-live", true)) + require.NoError(t, src.SetProperty("caps", gst.NewCapsFromString(raceRTPCaps))) + + rtpDepay, err := gst.NewElement("rtpopusdepay") + require.NoError(t, err) + opusDec, err := gst.NewElement("opusdec") + require.NoError(t, err) + + srcBin := p.NewBin(fmt.Sprintf("app_pre_%d", i)) + srcBin.SetEOSFunc(func() bool { return false }) + require.NoError(t, srcBin.AddElement(src.Element)) + require.NoError(t, srcBin.AddElements(rtpDepay, opusDec)) + require.NoError(t, srcBin.AddElements(buildAudioConverterChain(t)...)) + require.NoError(t, audioBin.AddSourceBin(srcBin)) + preAppSrcs[i] = src + } + + require.NoError(t, p.Link()) + + _, ok := p.UpgradeState(StateStarted) + require.True(t, ok) + + require.NoError(t, p.SetState(gst.StatePaused)) + + // Prepare the race source bin with full production chain: + // appsrc → rtpopusdepay → opusdec → audiorate → queue(leaky) → audioconvert → audioresample → capsfilter + raceSrc, err := app.NewAppSrc() + require.NoError(t, err) + raceSrc.SetArg("format", "time") + require.NoError(t, raceSrc.SetProperty("is-live", true)) + require.NoError(t, raceSrc.SetProperty("caps", gst.NewCapsFromString(raceRTPCaps))) + + rtpOpusDepay, err := gst.NewElement("rtpopusdepay") + require.NoError(t, err) + + opusDec, err := gst.NewElement("opusdec") + require.NoError(t, err) + + raceBin := p.NewBin("app_race_track") + raceBin.SetEOSFunc(func() bool { return false }) + require.NoError(t, raceBin.AddElement(raceSrc.Element)) + require.NoError(t, raceBin.AddElements(rtpOpusDepay, opusDec)) + require.NoError(t, raceBin.AddElements(buildAudioConverterChain(t)...)) + + // Install need-data callback BEFORE the race so we don't miss the first emission. + needData := make(chan struct{}, 1) + raceSrc.SetCallbacks(&app.SourceCallbacks{ + NeedDataFunc: func(_ *app.Source, _ uint) { + select { + case needData <- struct{}{}: + default: + } + }, + }) + + // Install a bus watch (processed by the GLib main loop, same as production). + // When the race bin reports PLAYING, break a "fuse" (channel close) so a + // separate pushSamples goroutine wakes up — matching the production path: + // bus watch → Playing(trackID) → opChan → track worker → writer.Playing() + // → fuse breaks → pushSamples goroutine wakes on separate OS thread + // Return false once done to remove the GSource from the default context. + var watchDone atomic.Bool + playingFuse := make(chan struct{}) + p.SetWatch(func(msg *gst.Message) bool { + if watchDone.Load() { + return false // remove watch + } + if msg.Type() == gst.MessageStateChanged { + _, newState := msg.ParseStateChanged() + if newState == gst.StatePlaying && strings.HasPrefix(msg.Source(), "app_race") { + watchDone.Store(true) + close(playingFuse) // break the fuse + return false // remove watch + } + } + return true + }) + + // Simulate pushSamples: a separate goroutine (own OS thread) waits for the + // playing fuse, then pushes buffers in a tight loop — matching production + // where pushSamples continuously dequeues from the jitter buffer and calls + // PushBuffer. This catches narrow flushing windows that a single push misses. + const pushCount = 50 + pushResult := make(chan gst.FlowReturn, 1) + go func() { + runtime.LockOSThread() + select { + case <-playingFuse: + case <-time.After(2 * time.Second): + pushResult <- gst.FlowError + return + } + for i := range pushCount { + buf := gst.NewBufferWithSize(96) + pts := gst.ClockTime(uint64(i) * uint64(time.Millisecond)) + buf.SetPresentationTimestamp(pts) + buf.SetDuration(gst.ClockTime(time.Millisecond)) + if flow := raceSrc.PushBuffer(buf); flow != gst.FlowOK { + pushResult <- flow + return + } + } + pushResult <- gst.FlowOK + }() + + // Start the GLib main loop (same as production's Pipeline.Run()). + // Use IdleAdd to confirm the loop is actually iterating before proceeding. + loopRunning := make(chan struct{}) + loopDone := make(chan struct{}) + glib.IdleAdd(func() bool { + select { + case <-loopRunning: + default: + close(loopRunning) + } + return false // remove idle source + }) + go func() { + runtime.LockOSThread() + p.loop.Run() + close(loopDone) + }() + + select { + case <-loopRunning: + result.loopRan = true + case <-time.After(time.Second): + t.Fatal("GLib main loop did not start within 1s") + } + + // Race: SetState(PLAYING) vs AddSourceBin + playingDone := make(chan error, 1) + addDone := make(chan error, 1) + + switch result.variant { + case variantBinAddFirst: + // Bin-add-first with 0-100μs head start + // (matches production: "track subscribes ~1ms before PAUSED→PLAYING") + go func() { + runtime.LockOSThread() + addDone <- audioBin.AddSourceBin(raceBin) + }() + time.Sleep(time.Duration(rand.Int63n(100)) * time.Microsecond) + go func() { + runtime.LockOSThread() + playingDone <- p.SetState(gst.StatePlaying) + }() + + case variantSimultaneous: + // Simultaneous: barrier-synchronized start + barrier := make(chan struct{}) + go func() { + runtime.LockOSThread() + <-barrier + playingDone <- p.SetState(gst.StatePlaying) + }() + go func() { + runtime.LockOSThread() + <-barrier + addDone <- audioBin.AddSourceBin(raceBin) + }() + close(barrier) + + case variantPlayingFirst: + // Playing-first with 0-100μs head start + go func() { + runtime.LockOSThread() + playingDone <- p.SetState(gst.StatePlaying) + }() + time.Sleep(time.Duration(rand.Int63n(100)) * time.Microsecond) + go func() { + runtime.LockOSThread() + addDone <- audioBin.AddSourceBin(raceBin) + }() + } + + err1 := <-playingDone + err2 := <-addDone + + if err1 != nil || err2 != nil { + result.setupError = true + watchDone.Store(true) + p.loop.Quit() + <-loopDone + _ = p.SetState(gst.StateNull) + return result + } + + // Wait for pushSamples goroutine result (fuse break + 50 buffer pushes) + select { + case flow := <-pushResult: + result.watchTriggered = true // fuse broke (bus watch fired) + switch flow { + case gst.FlowFlushing: + result.flowFlushing = true + case gst.FlowOK: + result.pushFlowOK = true // all 50 pushes succeeded + } + case <-time.After(2 * time.Second): + // Bus watch never saw PLAYING for the race bin + result.needDataMissing = true + } + + // Check need-data — if the streaming thread is alive, it should have + // fired by now (or shortly after PushBuffer woke it from cond_wait). + if !result.flowFlushing && !result.needDataMissing { + select { + case <-needData: + result.needDataReceived = true + case <-time.After(500 * time.Millisecond): + result.needDataMissing = true + } + } + + if result.flowFlushing || result.needDataMissing { + t.Logf("--- RACE HIT iter=%d (flowFlushing=%v, needDataMissing=%v) ---", + iteration, result.flowFlushing, result.needDataMissing) + t.Logf("appsrc state: %s", raceSrc.Element.GetCurrentState().String()) + t.Logf("race bin state: %s", raceBin.bin.GetCurrentState().String()) + t.Logf("pipeline state: %s", p.pipeline.GetCurrentState().String()) + } + + // Cleanup: quit loop first, then destroy pipeline + watchDone.Store(true) + p.loop.Quit() + <-loopDone + _ = raceSrc.EndStream() + for _, src := range preAppSrcs { + _ = src.EndStream() + } + _ = p.SetState(gst.StateNull) + time.Sleep(10 * time.Millisecond) + + return result +} diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 39a2d69a8..4c370dff9 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -133,7 +133,7 @@ func (h *Handler) Run() { } // Replay coordination: signal ready and get timing - if h.conf.IsReplay { + if !h.conf.Live { rctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) resp, err := h.ipcServiceClient.ReplayReady(rctx, &rpc.EgressReadyRequest{ EgressId: h.conf.Info.EgressId, diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index d4017bca8..24f366ea0 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -145,7 +145,7 @@ func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error return err } } else { - queue, err := gstreamer.BuildQueue(fmt.Sprintf("%s_queue", audioBinName), p.Latency.PipelineLatency, leakyQueue) + queue, err := gstreamer.BuildQueue(fmt.Sprintf("%s_queue", audioBinName), p.Latency.PipelineLatency, p.Live) if err != nil { return errors.ErrGstPipelineError(err) } @@ -220,8 +220,10 @@ func (b *AudioBin) buildSDKInput() error { return err } } - if err := b.addAudioTestSrcBin(); err != nil { - return err + if b.conf.Live { + if err := b.addAudioTestSrcBin(); err != nil { + return err + } } if err := b.addMixer(); err != nil { return err @@ -252,9 +254,14 @@ func (b *AudioBin) addAudioAppSrcBinLocked(ts *config.TrackSource) error { return false }) ts.AppSrc.SetArg("format", "time") - if err := ts.AppSrc.SetProperty("is-live", true); err != nil { + if err := ts.AppSrc.SetProperty("is-live", b.conf.Live); err != nil { return err } + if !b.conf.Live { + if err := ts.AppSrc.SetProperty("block", true); err != nil { + return err + } + } if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { return err } @@ -475,9 +482,126 @@ func (b *AudioBin) addMixer() error { subscribeForQoS(audioMixer) + if !b.conf.Live { + installMixerProbes(audioMixer) + } + return b.bin.AddElements(audioMixer, mixedCaps) } +// installTestSrcDisconnect sets up a probe on the mixer that sends EOS to the +// audiotestsrc once the first real audio buffer arrives from our appsrc path. +// The audiotestsrc is only needed for preroll — once real data is flowing, it +func installMixerProbes(mixer *gst.Element) { + // Install probes on sink pads as they're created (request pads) + mixer.Connect("pad-added", func(_ *gst.Element, pad *gst.Pad) { + name := pad.GetName() + var count uint64 + pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + count++ + buf := info.GetBuffer() + if buf != nil && (count <= 5 || count%100 == 0) { + logger.Infow("mixer sink probe", + "pad", name, + "count", count, + "pts", buf.PresentationTimestamp(), + "dur", buf.Duration(), + "size", buf.GetSize(), + ) + } + return gst.PadProbeOK + }) + pad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + if event := info.GetEvent(); event != nil { + logger.Infow("mixer sink event", "pad", name, "type", event.Type().String()) + } + return gst.PadProbeOK + }) + logger.Infow("mixer probe installed", "pad", name) + }) + + // Src pad is static — probe it directly + srcPad := mixer.GetStaticPad("src") + if srcPad != nil { + var srcCount uint64 + srcPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + srcCount++ + buf := info.GetBuffer() + if buf != nil && (srcCount <= 5 || srcCount%100 == 0) { + logger.Infow("mixer src probe", + "count", srcCount, + "pts", buf.PresentationTimestamp(), + "dur", buf.Duration(), + "size", buf.GetSize(), + ) + } + return gst.PadProbeOK + }) + srcPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + if event := info.GetEvent(); event != nil { + logger.Infow("mixer src event", "type", event.Type().String()) + } + return gst.PadProbeOK + }) + } +} + +func installQueueProbes(queue *gst.Element, name string) { + sinkPad := queue.GetStaticPad("sink") + if sinkPad != nil { + var sinkBufs uint64 + sinkPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + sinkBufs++ + buf := info.GetBuffer() + if buf != nil { + curTime, _ := queue.GetProperty("current-level-time") + curBufs, _ := queue.GetProperty("current-level-buffers") + logger.Infow("queue sink buf", + "queue", name, + "count", sinkBufs, + "pts", buf.PresentationTimestamp(), + "dur", buf.Duration(), + "size", buf.GetSize(), + "qTime", curTime, + "qBufs", curBufs, + ) + } + return gst.PadProbeOK + }) + sinkPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + if event := info.GetEvent(); event != nil { + logger.Infow("queue sink event", "queue", name, "type", event.Type().String(), "bufsSoFar", sinkBufs) + } + return gst.PadProbeOK + }) + } + + srcPad := queue.GetStaticPad("src") + if srcPad != nil { + var srcBufs uint64 + srcPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + srcBufs++ + buf := info.GetBuffer() + if buf != nil && (srcBufs <= 3 || srcBufs%100 == 0) { + logger.Infow("queue src buf", + "queue", name, + "count", srcBufs, + "pts", buf.PresentationTimestamp(), + "dur", buf.Duration(), + "size", buf.GetSize(), + ) + } + return gst.PadProbeOK + }) + srcPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + if event := info.GetEvent(); event != nil { + logger.Infow("queue src event", "queue", name, "type", event.Type().String(), "srcBufsSoFar", srcBufs) + } + return gst.PadProbeOK + }) + } +} + func (b *AudioBin) addEncoder() error { switch b.conf.AudioOutCodec { case types.MimeTypeOpus: @@ -532,6 +656,10 @@ func addAudioConverter(b *gstreamer.Bin, p *config.PipelineConfig, channel livek return err } + if !p.Live { + installQueueProbes(audioQueue, "audio_input_queue") + } + audioConvert, err := gst.NewElement("audioconvert") if err != nil { return errors.ErrGstPipelineError(err) diff --git a/pkg/pipeline/builder/file.go b/pkg/pipeline/builder/file.go index ed51a9f78..e4d4793a3 100644 --- a/pkg/pipeline/builder/file.go +++ b/pkg/pipeline/builder/file.go @@ -58,6 +58,11 @@ func BuildFileBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*gstr if err = sink.SetProperty("sync", false); err != nil { return nil, errors.ErrGstPipelineError(err) } + if !p.Live { + if err = sink.SetProperty("async", false); err != nil { + return nil, errors.ErrGstPipelineError(err) + } + } if err = b.AddElements(mux.GetElement(), sink); err != nil { return nil, err diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index c5641d905..b35130e08 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -49,9 +49,11 @@ type VideoBin struct { rawVideoTee *gst.Element } -// buildLeakyVideoQueue creates a leaky queue and attaches a monitor to track dropped buffers -func (b *VideoBin) buildLeakyVideoQueue(name string) (*gst.Element, error) { - queue, err := gstreamer.BuildQueue(name, b.conf.Latency.PipelineLatency, true) +// buildVideoQueue creates a queue for the video pipeline. For live sources the +// queue is leaky (drops old buffers when full) to handle real-time overrun. For +// non-live replay the queue is blocking so backpressure throttles the source. +func (b *VideoBin) buildVideoQueue(name string) (*gst.Element, error) { + queue, err := gstreamer.BuildQueue(name, b.conf.Latency.PipelineLatency, b.conf.Live) if err != nil { return nil, errors.ErrGstPipelineError(err) } @@ -97,7 +99,7 @@ func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error return tee.GetRequestPad("src_%u") } } else if len(p.GetEncodedOutputs()) > 0 { - queue, err := b.buildLeakyVideoQueue("video_queue") + queue, err := b.buildVideoQueue("video_queue") if err != nil { return err } @@ -281,7 +283,7 @@ func (b *VideoBin) buildWebInput() error { return errors.ErrGstPipelineError(err) } - videoQueue, err := b.buildLeakyVideoQueue("video_input_queue") + videoQueue, err := b.buildVideoQueue("video_input_queue") if err != nil { return err } @@ -384,9 +386,14 @@ func (b *VideoBin) buildAppSrcBin(ts *config.TrackSource, name string) (*gstream return false }) ts.AppSrc.SetArg("format", "time") - if err := ts.AppSrc.SetProperty("is-live", true); err != nil { + if err := ts.AppSrc.SetProperty("is-live", b.conf.Live); err != nil { return nil, errors.ErrGstPipelineError(err) } + if !b.conf.Live { + if err := ts.AppSrc.SetProperty("block", true); err != nil { + return nil, errors.ErrGstPipelineError(err) + } + } if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { return nil, err } @@ -725,7 +732,7 @@ func (b *VideoBin) addDecodedVideoSink() error { } func (b *VideoBin) addVideoConverter(bin *gstreamer.Bin) error { - videoQueue, err := b.buildLeakyVideoQueue("video_input_queue") + videoQueue, err := b.buildVideoQueue("video_input_queue") if err != nil { return err } diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index a3bdaa117..1f0be6a1d 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -104,7 +104,66 @@ func New(ctx context.Context, conf *config.PipelineConfig, ipcServiceClient ipc. ctx, span := tracer.Start(ctx, "Pipeline.New") defer span.End() + c := newController(conf, ipcServiceClient) + + // initialize gst + go func() { + _, span := tracer.Start(ctx, "gst.Init") + defer span.End() + gst.Init(nil) + gst.SetLogFunction(c.gstLog) + close(c.callbacks.GstReady) + }() + + // create source var err error + c.src, err = source.New(ctx, conf, c.callbacks) + if err != nil { + return nil, err + } + + // create pipeline + <-c.callbacks.GstReady + if err = c.BuildPipeline(); err != nil { + c.src.Close() + return nil, err + } + + return c, nil +} + +// NewWithSource creates a Controller with a pre-built source. This is used when +// the source is constructed externally (e.g. TestSource for non-live pipeline +// testing, or ReplaySource for offline export). The source must have already +// populated the PipelineConfig with track information before calling this. +func NewWithSource(ctx context.Context, conf *config.PipelineConfig, src source.Source) (*Controller, error) { + c := newController(conf, nil) + c.src = src + + // initialize gst + go func() { + gst.Init(nil) + gst.SetLogFunction(c.gstLog) + close(c.callbacks.GstReady) + }() + + // create pipeline + <-c.callbacks.GstReady + if err := c.BuildPipeline(); err != nil { + c.src.Close() + return nil, err + } + + return c, nil +} + +// Callbacks returns the pipeline callbacks. Sources that need to wait for +// GstReady before creating appsrc elements can use this. +func (c *Controller) Callbacks() *gstreamer.Callbacks { + return c.callbacks +} + +func newController(conf *config.PipelineConfig, ipcServiceClient ipc.EgressServiceClient) *Controller { c := &Controller{ PipelineConfig: conf, ipcServiceClient: ipcServiceClient, @@ -129,30 +188,7 @@ func New(ctx context.Context, conf *config.PipelineConfig, ipcServiceClient ipc. logger.Debugw("debug dot requested", "reason", reason) c.generateDotFile(reason) }) - - // initialize gst - go func() { - _, span := tracer.Start(ctx, "gst.Init") - defer span.End() - gst.Init(nil) - gst.SetLogFunction(c.gstLog) - close(c.callbacks.GstReady) - }() - - // create source - c.src, err = source.New(ctx, conf, c.callbacks) - if err != nil { - return nil, err - } - - // create pipeline - <-c.callbacks.GstReady - if err = c.BuildPipeline(); err != nil { - c.src.Close() - return nil, err - } - - return c, nil + return c } func (c *Controller) BuildPipeline() error { @@ -168,9 +204,9 @@ func (c *Controller) BuildPipeline() error { c.stopped.Break() return nil }) - if c.SourceType == types.SourceTypeSDK { + if sdkSrc, ok := c.src.(*source.SDKSource); ok { p.SetEOSFunc(func() bool { - c.src.(*source.SDKSource).CloseWriters() + sdkSrc.CloseWriters() return true }) } @@ -519,11 +555,11 @@ func (c *Controller) SendEOS(ctx context.Context, reason string) { case livekit.EgressStatus_EGRESS_ACTIVE: c.Info.UpdateStatus(livekit.EgressStatus_EGRESS_ENDING) - _, _ = c.ipcServiceClient.HandlerUpdate(ctx, c.Info) + c.sendHandlerUpdate(ctx, c.Info) c.sendEOS() case livekit.EgressStatus_EGRESS_ENDING: - _, _ = c.ipcServiceClient.HandlerUpdate(ctx, c.Info) + c.sendHandlerUpdate(ctx, c.Info) c.sendEOS() case livekit.EgressStatus_EGRESS_LIMIT_REACHED: @@ -856,7 +892,7 @@ func (c *Controller) updateStartTime(startedAt int64) { if c.Info.Status == livekit.EgressStatus_EGRESS_STARTING { c.Info.UpdateStatus(livekit.EgressStatus_EGRESS_ACTIVE) - _, _ = c.ipcServiceClient.HandlerUpdate(context.Background(), c.Info) + c.sendHandlerUpdate(context.Background(), c.Info) } } @@ -894,7 +930,13 @@ func (c *Controller) streamUpdated(ctx context.Context) { } } - _, _ = c.ipcServiceClient.HandlerUpdate(ctx, c.Info) + c.sendHandlerUpdate(ctx, c.Info) +} + +func (c *Controller) sendHandlerUpdate(ctx context.Context, info *livekit.EgressInfo) { + if c.ipcServiceClient != nil { + _, _ = c.ipcServiceClient.HandlerUpdate(ctx, info) + } } func (c *Controller) updateEndTime() { diff --git a/pkg/pipeline/source/testfeeder/feeder.go b/pkg/pipeline/source/testfeeder/feeder.go new file mode 100644 index 000000000..f8ddce585 --- /dev/null +++ b/pkg/pipeline/source/testfeeder/feeder.go @@ -0,0 +1,235 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testfeeder provides a non-live media file feeder for GStreamer appsrc +// elements. It reads encoded media from files (H.264 annexb, VP8/VP9 IVF, Opus +// OGG), packetizes the frames into RTP packets, and pushes them to an appsrc as +// fast as the pipeline will accept. This is used to test the egress GStreamer +// pipeline in non-live (faster-than-realtime) mode without needing a live +// WebRTC source or the full replay infrastructure. +package testfeeder + +import ( + "fmt" + "io" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + + "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/logger" +) + +const ( + defaultMTU = 1200 + videoClockRate = 90000 + opusClockRate = 48000 + defaultVideoFPS = 30 + defaultOpusFrame = 20 * time.Millisecond +) + +// FrameReader reads encoded media frames from a file. +// Each call to NextFrame returns the raw encoded data for one frame +// and the frame's duration in RTP clock rate units (samples). +type FrameReader interface { + // NextFrame returns the next frame's payload and its duration in RTP + // timestamp units (samples at the codec clock rate). Returns io.EOF + // when all frames have been read. + NextFrame() (payload []byte, samples uint32, err error) + + // ClockRate returns the RTP clock rate for this media type. + ClockRate() uint32 + + io.Closer +} + +// TrackFeeder reads frames from a FrameReader, packetizes them into RTP, and +// pushes the resulting buffers to a GStreamer appsrc element. When the appsrc +// is configured with is-live=false, PushBuffer blocks on backpressure, +// naturally throttling the feeder to the pipeline's processing speed. +type TrackFeeder struct { + src *app.Source + reader FrameReader + packetizer rtp.Packetizer + codec types.MimeType + startTS uint32 + currentTS uint32 +} + +// TrackFeederParams configures a TrackFeeder. +type TrackFeederParams struct { + // AppSrc is the GStreamer appsrc element to push buffers to. + AppSrc *app.Source + + // MimeType selects the codec (types.MimeTypeH264, MimeTypeVP8, MimeTypeOpus, etc.). + MimeType types.MimeType + + // PayloadType is the RTP payload type number (e.g. 96 for H264, 111 for Opus). + PayloadType uint8 + + // SSRC for the RTP stream. Use any fixed value for testing. + SSRC uint32 + + // Reader is the frame source. Use NewH264Reader, NewIVFReader, or NewOGGReader. + Reader FrameReader +} + +// NewTrackFeeder creates a TrackFeeder that reads from the given FrameReader +// and pushes RTP-packetized buffers to the appsrc. +func NewTrackFeeder(params TrackFeederParams) (*TrackFeeder, error) { + payloader, err := payloaderForCodec(params.MimeType) + if err != nil { + return nil, err + } + + clockRate := params.Reader.ClockRate() + + packetizer := rtp.NewPacketizer( + defaultMTU, + params.PayloadType, + params.SSRC, + payloader, + rtp.NewRandomSequencer(), + clockRate, + ) + + return &TrackFeeder{ + src: params.AppSrc, + reader: params.Reader, + packetizer: packetizer, + codec: params.MimeType, + }, nil +} + +// Feed reads all frames from the reader, packetizes them, and pushes them to +// the appsrc. It blocks until all frames are consumed (io.EOF from reader) or +// an error occurs. On completion it sends an EOS event to the appsrc. +// +// With is-live=false on the appsrc, PushBuffer provides natural backpressure: +// when the downstream pipeline is busy, the push blocks, so no rate-limiting +// code is needed. +func (f *TrackFeeder) Feed() error { + defer f.sendEOS() + + // Log appsrc properties + if maxBuf, err := f.src.GetProperty("max-buffers"); err == nil { + logger.Infow("appsrc config", + "max-buffers", maxBuf, + "is-live", func() interface{} { v, _ := f.src.GetProperty("is-live"); return v }(), + "max-bytes", func() interface{} { v, _ := f.src.GetProperty("max-bytes"); return v }(), + "current-level-buffers", func() interface{} { v, _ := f.src.GetProperty("current-level-buffers"); return v }(), + ) + } + + clockRate := f.reader.ClockRate() + firstFrame := true + frameCount := 0 + packetCount := 0 + totalBytes := 0 + start := time.Now() + + for { + payload, samples, err := f.reader.NextFrame() + if err == io.EOF { + logger.Infow("feeder done", + "frames", frameCount, + "packets", packetCount, + "bytes", totalBytes, + "elapsed", time.Since(start), + "codec", f.codec, + ) + return nil + } + if err != nil { + return fmt.Errorf("reading frame: %w", err) + } + + frameCount++ + packets := f.packetizer.Packetize(payload, samples) + for _, pkt := range packets { + if firstFrame { + // Capture the actual RTP timestamp from the first packet. + // The packetizer starts with a random base timestamp. + f.startTS = pkt.Timestamp + firstFrame = false + } + packetCount++ + if err := f.pushRTPPacket(pkt, clockRate); err != nil { + return fmt.Errorf("pushing packet seq=%d: %w", pkt.SequenceNumber, err) + } + totalBytes += len(pkt.Payload) + } + + f.currentTS += samples + + if frameCount%100 == 0 { + pts := rtpToDuration(f.currentTS, clockRate) + logger.Debugw("feeder progress", + "frames", frameCount, + "packets", packetCount, + "mediaPTS", pts, + "elapsed", time.Since(start), + ) + } + } +} + +// pushRTPPacket marshals an RTP packet, creates a GstBuffer with the +// appropriate PTS, and pushes it to the appsrc. +func (f *TrackFeeder) pushRTPPacket(pkt *rtp.Packet, clockRate uint32) error { + buf, err := pkt.Marshal() + if err != nil { + return fmt.Errorf("marshaling RTP packet: %w", err) + } + + pts := rtpToDuration(pkt.Timestamp-f.startTS, clockRate) + + b := gst.NewBufferFromBytes(buf) + b.SetPresentationTimestamp(gst.ClockTime(uint64(pts))) + + flow := f.src.PushBuffer(b) + if flow != gst.FlowOK { + return fmt.Errorf("appsrc push returned %v", flow) + } + return nil +} + +func (f *TrackFeeder) sendEOS() { + f.src.EndStream() +} + +// rtpToDuration converts an RTP timestamp delta to a time.Duration. +func rtpToDuration(rtpTS uint32, clockRate uint32) time.Duration { + return time.Duration(float64(rtpTS) / float64(clockRate) * float64(time.Second)) +} + +func payloaderForCodec(mime types.MimeType) (rtp.Payloader, error) { + switch mime { + case types.MimeTypeH264: + return &codecs.H264Payloader{}, nil + case types.MimeTypeVP8: + return &codecs.VP8Payloader{ + EnablePictureID: true, + }, nil + case types.MimeTypeVP9: + return &codecs.VP9Payloader{}, nil + case types.MimeTypeOpus: + return &codecs.OpusPayloader{}, nil + default: + return nil, fmt.Errorf("unsupported codec for test feeder: %s", mime) + } +} diff --git a/pkg/pipeline/source/testfeeder/reader_h264.go b/pkg/pipeline/source/testfeeder/reader_h264.go new file mode 100644 index 000000000..a4041a0b7 --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_h264.go @@ -0,0 +1,81 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import ( + "io" + "os" + + "github.com/pion/webrtc/v4/pkg/media/h264reader" +) + +// H264Reader reads H.264 NAL units from an Annex B bitstream file and presents +// them as frames suitable for RTP packetization. +// +// Each call to NextFrame returns a complete NAL unit. The frame duration is +// fixed at (clockRate / fps) samples, which is correct for constant-framerate +// content. For VFR content the duration is approximate, but sufficient for +// pipeline testing. +type H264Reader struct { + reader *h264reader.H264Reader + file *os.File + fps int +} + +// NewH264Reader opens an H.264 Annex B file and returns a FrameReader. +// fps sets the assumed frame rate for timestamp advancement. +func NewH264Reader(path string, fps int) (*H264Reader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + h, err := h264reader.NewReader(f) + if err != nil { + f.Close() + return nil, err + } + + if fps <= 0 { + fps = defaultVideoFPS + } + + return &H264Reader{ + reader: h, + file: f, + fps: fps, + }, nil +} + +func (r *H264Reader) NextFrame() ([]byte, uint32, error) { + nal, err := r.reader.NextNAL() + if err != nil { + if err == io.EOF { + return nil, 0, io.EOF + } + return nil, 0, err + } + + samples := uint32(videoClockRate / r.fps) + return nal.Data, samples, nil +} + +func (r *H264Reader) ClockRate() uint32 { + return videoClockRate +} + +func (r *H264Reader) Close() error { + return r.file.Close() +} diff --git a/pkg/pipeline/source/testfeeder/reader_ivf.go b/pkg/pipeline/source/testfeeder/reader_ivf.go new file mode 100644 index 000000000..6faed1a90 --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_ivf.go @@ -0,0 +1,93 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import ( + "io" + "os" + + "github.com/pion/webrtc/v4/pkg/media/ivfreader" +) + +// IVFReader reads VP8 or VP9 frames from an IVF container file. +// The IVF container provides per-frame timestamps, so duration is computed from +// the difference between consecutive frame timestamps (converted to clock rate +// units via the IVF timebase). +type IVFReader struct { + reader *ivfreader.IVFReader + file *os.File + header *ivfreader.IVFFileHeader + lastPTS uint64 + firstRead bool +} + +// NewIVFReader opens an IVF file and returns a FrameReader. +func NewIVFReader(path string) (*IVFReader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + reader, header, err := ivfreader.NewWith(f) + if err != nil { + f.Close() + return nil, err + } + + return &IVFReader{ + reader: reader, + file: f, + header: header, + firstRead: true, + }, nil +} + +func (r *IVFReader) NextFrame() ([]byte, uint32, error) { + payload, frameHeader, err := r.reader.ParseNextFrame() + if err != nil { + if err == io.EOF { + return nil, 0, io.EOF + } + return nil, 0, err + } + + var samples uint32 + if r.firstRead { + // First frame: use default frame duration based on the timebase. + // timebaseDenominator is fps (e.g., 24), timebaseNumerator is typically 1. + samples = videoClockRate * uint32(r.header.TimebaseNumerator) / uint32(r.header.TimebaseDenominator) + r.firstRead = false + } else { + // Duration = difference between this frame's timestamp and the last. + // IVF timestamps are in timebase units (numerator/denominator seconds). + diff := frameHeader.Timestamp - r.lastPTS + samples = uint32(diff * uint64(videoClockRate) * uint64(r.header.TimebaseNumerator) / uint64(r.header.TimebaseDenominator)) + if samples == 0 { + // Fallback for zero-duration frames. + samples = videoClockRate * uint32(r.header.TimebaseNumerator) / uint32(r.header.TimebaseDenominator) + } + } + r.lastPTS = frameHeader.Timestamp + + return payload, samples, nil +} + +func (r *IVFReader) ClockRate() uint32 { + return videoClockRate +} + +func (r *IVFReader) Close() error { + return r.file.Close() +} diff --git a/pkg/pipeline/source/testfeeder/reader_ogg.go b/pkg/pipeline/source/testfeeder/reader_ogg.go new file mode 100644 index 000000000..a069469ba --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_ogg.go @@ -0,0 +1,148 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import ( + "encoding/binary" + "io" + "os" +) + +const ( + oggPageHeaderLen = 27 + oggHeaderSig = "OggS" + + // Opus uses 20ms frames at 48kHz = 960 samples per frame. + opusSamplesPerFrame = 960 +) + +// OGGReader reads individual Opus frames from an OGG container file. +// OGG pages can contain multiple Opus packets; this reader splits them +// using the OGG segment table so each NextFrame call returns exactly one +// Opus frame suitable for RTP packetization. +type OGGReader struct { + stream io.ReadCloser + + // Buffer of Opus packets extracted from the current OGG page. + // Each call to NextFrame pops one packet from this buffer. + packets [][]byte +} + +// NewOGGReader opens an OGG file containing Opus audio and returns a FrameReader. +func NewOGGReader(path string) (*OGGReader, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + + r := &OGGReader{stream: f} + + // Skip the OGG header pages (OpusHead + OpusTags). + // These are identified by the beginning-of-stream flag or by having + // granule position 0. We just skip until we find a page with audio data. + for { + packets, granule, err := r.readPage() + if err != nil { + f.Close() + return nil, err + } + // Audio pages have granule position > 0 + if granule > 0 { + r.packets = packets + break + } + } + + return r, nil +} + +func (r *OGGReader) NextFrame() ([]byte, uint32, error) { + for len(r.packets) == 0 { + packets, _, err := r.readPage() + if err != nil { + if err == io.EOF { + return nil, 0, io.EOF + } + return nil, 0, err + } + r.packets = packets + } + + pkt := r.packets[0] + r.packets = r.packets[1:] + return pkt, opusSamplesPerFrame, nil +} + +func (r *OGGReader) ClockRate() uint32 { + return opusClockRate +} + +func (r *OGGReader) Close() error { + return r.stream.Close() +} + +// readPage reads one OGG page and splits its payload into individual packets +// using the segment table. Returns the packets and the page's granule position. +func (r *OGGReader) readPage() ([][]byte, uint64, error) { + // Read page header (27 bytes) + header := make([]byte, oggPageHeaderLen) + if _, err := io.ReadFull(r.stream, header); err != nil { + return nil, 0, err + } + + if string(header[0:4]) != oggHeaderSig { + return nil, 0, io.ErrUnexpectedEOF + } + + granule := binary.LittleEndian.Uint64(header[6:14]) + segCount := int(header[26]) + + // Read segment table + segTable := make([]byte, segCount) + if _, err := io.ReadFull(r.stream, segTable); err != nil { + return nil, 0, err + } + + // Compute total payload size and read it + var totalSize int + for _, s := range segTable { + totalSize += int(s) + } + payload := make([]byte, totalSize) + if _, err := io.ReadFull(r.stream, payload); err != nil { + return nil, 0, err + } + + // Split payload into packets using the segment table. + // A packet spans consecutive segments; it ends when a segment is < 255. + var packets [][]byte + offset := 0 + packetStart := 0 + for _, s := range segTable { + offset += int(s) + if s < 255 { + // End of packet + if offset > packetStart { + packets = append(packets, payload[packetStart:offset]) + } + packetStart = offset + } + } + // Handle trailing packet that ends exactly on a 255-byte boundary + // (no terminating short segment). This is a continuation that spans + // to the next page — skip it for now. + + return packets, granule, nil +} diff --git a/pkg/pipeline/source/testfeeder/source.go b/pkg/pipeline/source/testfeeder/source.go new file mode 100644 index 000000000..a20fbaf1b --- /dev/null +++ b/pkg/pipeline/source/testfeeder/source.go @@ -0,0 +1,272 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import ( + "fmt" + "sync" + "time" + + "github.com/frostbyte73/core" + "github.com/go-gst/go-gst/gst" + "github.com/go-gst/go-gst/gst/app" + "github.com/pion/webrtc/v4" + + "github.com/livekit/egress/pkg/config" + "github.com/livekit/egress/pkg/gstreamer" + "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/logger" + lksdk "github.com/livekit/server-sdk-go/v2" +) + +// TestSource implements the source.Source interface by feeding pre-encoded media +// files through the GStreamer pipeline at maximum speed. It creates TrackSource +// objects, populates the PipelineConfig, and runs TrackFeeder goroutines after +// the pipeline starts. +// +// GStreamer must be initialized (gst.Init) before calling NewTestSource, since +// it creates appsrc elements. +// +// Usage: +// +// gst.Init(nil) +// ts, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ +// {Path: "audio.ogg", MimeType: types.MimeTypeOpus, PayloadType: 111}, +// {Path: "video.h264", MimeType: types.MimeTypeH264, PayloadType: 96, FPS: 30}, +// }) +// // conf.AudioTracks, conf.VideoTrack, etc. are now populated. +// // Build the pipeline, then call ts.Start() to begin feeding. +type TestSource struct { + conf *config.PipelineConfig + callbacks *gstreamer.Callbacks + + tracks []trackEntry + feeders []*TrackFeeder + startedAt int64 + + endRecording core.Fuse +} + +type trackEntry struct { + def TrackDef + ts *config.TrackSource + reader FrameReader +} + +// TrackDef describes a media file to feed into the pipeline. +type TrackDef struct { + // Path to the media file (H.264 annexb, VP8/VP9 IVF, or Opus OGG). + Path string + + // MimeType of the media (types.MimeTypeH264, MimeTypeVP8, MimeTypeOpus, etc.). + MimeType types.MimeType + + // PayloadType is the RTP payload type number. + PayloadType uint8 + + // FPS is the assumed frame rate for video files. Ignored for audio. + // Defaults to 30 if zero. + FPS int + + // TrackID is an optional identifier. Auto-generated if empty. + TrackID string +} + +// NewTestSource creates a TestSource that populates the PipelineConfig with +// TrackSource objects for each track definition. GStreamer must already be +// initialized. After this returns, the config is ready for BuildPipeline. +// Call Start() after the pipeline is built to begin feeding data. +func NewTestSource(conf *config.PipelineConfig, defs []TrackDef) (*TestSource, error) { + s := &TestSource{ + conf: conf, + } + + conf.Live = false + conf.SourceType = types.SourceTypeSDK + + for i, def := range defs { + if def.TrackID == "" { + def.TrackID = fmt.Sprintf("test_track_%d", i) + } + + reader, err := openReader(def) + if err != nil { + s.closeReaders() + return nil, fmt.Errorf("opening %s: %w", def.Path, err) + } + + src, err := gst.NewElementWithName("appsrc", fmt.Sprintf("app_%s", def.TrackID)) + if err != nil { + s.closeReaders() + return nil, fmt.Errorf("creating appsrc: %w", err) + } + + ts := &config.TrackSource{ + TrackID: def.TrackID, + MimeType: def.MimeType, + PayloadType: webrtc.PayloadType(def.PayloadType), + ClockRate: reader.ClockRate(), + AppSrc: app.SrcFromElement(src), + } + + switch def.MimeType { + case types.MimeTypeOpus, types.MimeTypePCMU, types.MimeTypePCMA: + ts.TrackKind = lksdk.TrackKindAudio + conf.AudioEnabled = true + conf.AudioTranscoding = true + if conf.AudioOutCodec == "" { + if def.MimeType == types.MimeTypePCMU || def.MimeType == types.MimeTypePCMA { + conf.AudioOutCodec = types.MimeTypeOpus + } else { + conf.AudioOutCodec = def.MimeType + } + } + conf.AudioTracks = append(conf.AudioTracks, ts) + + case types.MimeTypeH264, types.MimeTypeVP8, types.MimeTypeVP9: + ts.TrackKind = lksdk.TrackKindVideo + conf.VideoEnabled = true + conf.VideoInCodec = def.MimeType + if conf.VideoOutCodec == "" { + conf.VideoOutCodec = def.MimeType + } + if conf.VideoInCodec != conf.VideoOutCodec { + conf.VideoDecoding = true + if len(conf.GetEncodedOutputs()) > 0 { + conf.VideoEncoding = true + } + } + conf.VideoTrack = ts + + default: + s.closeReaders() + return nil, fmt.Errorf("unsupported mime type: %s", def.MimeType) + } + + s.tracks = append(s.tracks, trackEntry{ + def: def, + ts: ts, + reader: reader, + }) + } + + return s, nil +} + +// --- source.Source interface --- + +// StartRecording returns a channel that closes immediately, signaling to the +// controller that the source is ready. It also starts feeding all tracks +// concurrently in background goroutines. EndRecording fires when all feeders +// have finished and sent EOS. +func (s *TestSource) StartRecording() <-chan struct{} { + s.startedAt = time.Now().UnixNano() + + // Launch feeders in a goroutine — they will wait for the pipeline to + // reach PLAYING state before pushing any data, so StartRecording can + // return immediately to unblock the controller's Run loop. + go s.feedAll() + + ch := make(chan struct{}) + close(ch) + return ch +} + +func (s *TestSource) feedAll() { + // Wait for the pipeline to reach PAUSED before pushing data. + // The audiotestsrc (is-live=true) prerolls the pipeline. + <-s.callbacks.PipelinePaused() + + var wg sync.WaitGroup + for i := range s.tracks { + te := &s.tracks[i] + + feeder, err := NewTrackFeeder(TrackFeederParams{ + AppSrc: te.ts.AppSrc, + MimeType: te.def.MimeType, + PayloadType: te.def.PayloadType, + SSRC: uint32(1000 + i), + Reader: te.reader, + }) + if err != nil { + logger.Errorw("failed to create feeder", err, "trackID", te.def.TrackID) + continue + } + s.feeders = append(s.feeders, feeder) + + wg.Add(1) + go func() { + defer wg.Done() + if err := feeder.Feed(); err != nil { + logger.Errorw("feeder error", err, "trackID", te.def.TrackID) + } + }() + } + + wg.Wait() + logger.Infow("all test feeders finished, EOS sent on all appsrc elements") + s.endRecording.Break() +} + +func (s *TestSource) EndRecording() <-chan struct{} { + return s.endRecording.Watch() +} + +func (s *TestSource) GetStartedAt() int64 { + return s.startedAt +} + +func (s *TestSource) GetEndedAt() int64 { + return time.Now().UnixNano() +} + +func (s *TestSource) Close() { + s.closeReaders() +} + +// --- source.TimeAware interface --- + +func (s *TestSource) SetTimeProvider(_ gstreamer.TimeProvider) { + // No-op. Non-live pipeline doesn't need time feedback. +} + +// SetCallbacks sets the pipeline callbacks. Must be called before the pipeline +// starts (before Run). The controller exposes callbacks via Callbacks(). +func (s *TestSource) SetCallbacks(cb *gstreamer.Callbacks) { + s.callbacks = cb +} + +// --- helpers --- + +func (s *TestSource) closeReaders() { + for _, te := range s.tracks { + if te.reader != nil { + te.reader.Close() + } + } +} + +func openReader(def TrackDef) (FrameReader, error) { + switch def.MimeType { + case types.MimeTypeH264: + return NewH264Reader(def.Path, def.FPS) + case types.MimeTypeVP8, types.MimeTypeVP9: + return NewIVFReader(def.Path) + case types.MimeTypeOpus: + return NewOGGReader(def.Path) + default: + return nil, fmt.Errorf("no reader for %s", def.MimeType) + } +} diff --git a/pkg/pipeline/source/testfeeder/source_test.go b/pkg/pipeline/source/testfeeder/source_test.go new file mode 100644 index 000000000..5f25d25a0 --- /dev/null +++ b/pkg/pipeline/source/testfeeder/source_test.go @@ -0,0 +1,153 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package testfeeder_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/go-gst/go-gst/gst" + "github.com/stretchr/testify/require" + + "github.com/livekit/egress/pkg/config" + "github.com/livekit/egress/pkg/pipeline" + "github.com/livekit/egress/pkg/pipeline/source/testfeeder" + "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/observability/storageobs" +) + +func init() { + os.Setenv("GST_DEBUG", "audiomixer:6,aggregator:6,queue:5,audiotestsrc:5") + logger.InitFromConfig(&logger.Config{Level: "debug"}, "testfeeder") +} + +// mediaSamplesDir returns the path to the media-samples directory. +// Set MEDIA_SAMPLES_DIR env var to override, otherwise defaults to the +// cloud-egress media-samples relative to this repo. +func mediaSamplesDir(t *testing.T) string { + if dir := os.Getenv("MEDIA_SAMPLES_DIR"); dir != "" { + return dir + } + // Default: assume egress and cloud-egress are sibling dirs + dir := filepath.Join("..", "..", "..", "..", "..", "cloud-egress", "media-samples") + abs, err := filepath.Abs(dir) + require.NoError(t, err) + if _, err := os.Stat(abs); err != nil { + t.Skipf("media-samples not found at %s (set MEDIA_SAMPLES_DIR)", abs) + } + return abs +} + +// newTestPipelineConfig creates a minimal PipelineConfig suitable for pipeline +// construction. Track info is populated by TestSource; this sets up everything +// else the pipeline needs (latency, output config, etc.). +func newTestPipelineConfig(t *testing.T, outputPath string, outputType types.OutputType) *config.PipelineConfig { + t.Helper() + + tmpDir := t.TempDir() + + p := &config.PipelineConfig{ + BaseConfig: config.BaseConfig{ + Logging: &logger.Config{Level: "debug"}, + Latency: config.LatencyConfig{ + PipelineLatency: 3 * time.Second, + }, + }, + TmpDir: tmpDir, + Outputs: make(map[types.EgressType][]config.OutputConfig), + StorageReporter: storageobs.NewNoopProjectReporter(), + } + + p.AudioConfig = config.AudioConfig{ + AudioBitrate: 128, + AudioFrequency: 48000, + } + + p.Info = &livekit.EgressInfo{ + EgressId: "test-feeder-integration", + Status: livekit.EgressStatus_EGRESS_STARTING, + StartedAt: time.Now().UnixNano(), + UpdatedAt: time.Now().UnixNano(), + } + + p.Outputs[types.EgressTypeFile] = []config.OutputConfig{ + &config.FileConfig{ + FileInfo: &livekit.FileInfo{}, + LocalFilepath: outputPath, + StorageFilepath: outputPath, + DisableManifest: true, + }, + } + p.GetFileConfig().OutputType = outputType + + return p +} + +// TestAudioOnlyOGG tests the full non-live pipeline with a single Opus audio +// track producing an OGG file. This is the simplest case: one appsrc, one +// depayloader, one decoder, one encoder, one muxer, one file sink. +func TestAudioOnlyOGG(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-output.ogg" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) + + // Initialize GStreamer before creating source (appsrc needs it) + gst.Init(nil) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + Path: filepath.Join(samples, "SolLevante.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + }) + require.NoError(t, err) + + // Verify config was populated + require.True(t, conf.AudioEnabled) + require.False(t, conf.Live) + require.Len(t, conf.AudioTracks, 1) + + // Create pipeline via controller + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + // Wire callbacks so feeders know when pipeline is ready + src.SetCallbacks(ctrl.Callbacks()) + + // Run pipeline to completion — this blocks until EOS. + // The controller calls src.StartRecording() which starts the feeders. + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + // Verify output file exists and has content + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} diff --git a/pkg/pipeline/watch.go b/pkg/pipeline/watch.go index 552b9fc67..74a127b31 100644 --- a/pkg/pipeline/watch.go +++ b/pkg/pipeline/watch.go @@ -232,7 +232,9 @@ func (c *Controller) handleMessageError(gErr *gst.GError) error { if message == msgStreamingNotNegotiated { // send eosSent to app src logger.Debugw("streaming stopped", "name", name) - c.src.(*source.SDKSource).StreamStopped(name) + if sdkSrc, ok := c.src.(*source.SDKSource); ok { + sdkSrc.StreamStopped(name) + } return nil } @@ -281,7 +283,9 @@ func (c *Controller) handleMessageStateChanged(msg *gst.Message) { trackID := s[4:] logger.Debugw("appsrc state change", "trackID", trackID, "oldState", oldState.String(), "newState", newState.String()) if newState == gst.StatePlaying { - c.src.(*source.SDKSource).Playing(trackID) + if sdkSrc, ok := c.src.(*source.SDKSource); ok { + sdkSrc.Playing(trackID) + } } return } From 71e219ba47c6a3a722d15c915be7bdd839f93fea Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Wed, 8 Apr 2026 16:31:13 +0200 Subject: [PATCH 2/9] More test cases to cover video tracks, participant recordings, audio mixing, different feeding tempo --- pkg/pipeline/source/testfeeder/feeder.go | 39 +- .../source/testfeeder/reader_gapping.go | 80 ++++ pkg/pipeline/source/testfeeder/reader_slow.go | 44 ++ .../source/testfeeder/reader_stalling.go | 85 ++++ pkg/pipeline/source/testfeeder/source.go | 17 +- pkg/pipeline/source/testfeeder/source_test.go | 415 ++++++++++++++++++ 6 files changed, 675 insertions(+), 5 deletions(-) create mode 100644 pkg/pipeline/source/testfeeder/reader_gapping.go create mode 100644 pkg/pipeline/source/testfeeder/reader_slow.go create mode 100644 pkg/pipeline/source/testfeeder/reader_stalling.go diff --git a/pkg/pipeline/source/testfeeder/feeder.go b/pkg/pipeline/source/testfeeder/feeder.go index f8ddce585..2edd27715 100644 --- a/pkg/pipeline/source/testfeeder/feeder.go +++ b/pkg/pipeline/source/testfeeder/feeder.go @@ -21,6 +21,7 @@ package testfeeder import ( + "errors" "fmt" "io" "time" @@ -68,6 +69,7 @@ type TrackFeeder struct { codec types.MimeType startTS uint32 currentTS uint32 + ptsOffset time.Duration // accumulated offset from GAP events } // TrackFeederParams configures a TrackFeeder. @@ -154,6 +156,21 @@ func (f *TrackFeeder) Feed() error { ) return nil } + var gapErr *ErrGap + if errors.As(err, &gapErr) { + pts := rtpToDuration(f.currentTS, clockRate) + logger.Infow("feeder sending GAP event", + "pts", pts, + "gapDuration", gapErr.Duration, + "framesBeforeGap", frameCount, + ) + if err := f.sendGapEvent(pts+f.ptsOffset, gapErr.Duration); err != nil { + return fmt.Errorf("sending GAP event: %w", err) + } + // Accumulate the gap so subsequent buffer PTS values are shifted forward. + f.ptsOffset += gapErr.Duration + continue + } if err != nil { return fmt.Errorf("reading frame: %w", err) } @@ -196,7 +213,7 @@ func (f *TrackFeeder) pushRTPPacket(pkt *rtp.Packet, clockRate uint32) error { return fmt.Errorf("marshaling RTP packet: %w", err) } - pts := rtpToDuration(pkt.Timestamp-f.startTS, clockRate) + pts := rtpToDuration(pkt.Timestamp-f.startTS, clockRate) + f.ptsOffset b := gst.NewBufferFromBytes(buf) b.SetPresentationTimestamp(gst.ClockTime(uint64(pts))) @@ -212,6 +229,26 @@ func (f *TrackFeeder) sendEOS() { f.src.EndStream() } +// sendGapEvent pushes a GAP event downstream from the appsrc, telling the +// mixer to insert silence for this track over the given duration. +// +// GAP is a downstream serialized event. We push it via Pad.PushEvent on the +// appsrc's src pad (not SendEvent, which is for upstream events). +func (f *TrackFeeder) sendGapEvent(pts time.Duration, duration time.Duration) error { + event := gst.NewGapEvent( + gst.ClockTime(uint64(pts)), + gst.ClockTime(uint64(duration)), + ) + srcPad := f.src.GetStaticPad("src") + if srcPad == nil { + return fmt.Errorf("appsrc has no src pad") + } + if !srcPad.PushEvent(event) { + return fmt.Errorf("failed to push GAP event on appsrc src pad") + } + return nil +} + // rtpToDuration converts an RTP timestamp delta to a time.Duration. func rtpToDuration(rtpTS uint32, clockRate uint32) time.Duration { return time.Duration(float64(rtpTS) / float64(clockRate) * float64(time.Second)) diff --git a/pkg/pipeline/source/testfeeder/reader_gapping.go b/pkg/pipeline/source/testfeeder/reader_gapping.go new file mode 100644 index 000000000..786219fb9 --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_gapping.go @@ -0,0 +1,80 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import ( + "fmt" + "io" + "time" +) + +// ErrGap is returned by a FrameReader to signal a gap in the media data. +// The feeder should send a GStreamer GAP event for the specified duration +// instead of pushing a buffer, then continue reading. +type ErrGap struct { + Duration time.Duration +} + +func (e *ErrGap) Error() string { + return fmt.Sprintf("gap: %v", e.Duration) +} + +// GappingReader wraps a FrameReader and introduces a gap after a fixed number +// of frames. After delivering gapAfter frames, it returns an ErrGap with the +// specified duration. After the gap, it continues reading from the inner reader +// until EOF. +type GappingReader struct { + inner FrameReader + gapAfter int + gapDuration time.Duration + delivered int + gapSent bool +} + +// NewGappingReader wraps an existing FrameReader. After gapAfter frames, the +// next call to NextFrame returns ErrGap with the given duration. Subsequent +// calls continue reading from inner. +func NewGappingReader(inner FrameReader, gapAfter int, gapDuration time.Duration) *GappingReader { + return &GappingReader{ + inner: inner, + gapAfter: gapAfter, + gapDuration: gapDuration, + } +} + +func (r *GappingReader) NextFrame() ([]byte, uint32, error) { + if r.delivered >= r.gapAfter && !r.gapSent { + r.gapSent = true + return nil, 0, &ErrGap{Duration: r.gapDuration} + } + + payload, samples, err := r.inner.NextFrame() + if err != nil { + if err == io.EOF { + return nil, 0, io.EOF + } + return nil, 0, err + } + r.delivered++ + return payload, samples, nil +} + +func (r *GappingReader) ClockRate() uint32 { + return r.inner.ClockRate() +} + +func (r *GappingReader) Close() error { + return r.inner.Close() +} diff --git a/pkg/pipeline/source/testfeeder/reader_slow.go b/pkg/pipeline/source/testfeeder/reader_slow.go new file mode 100644 index 000000000..2183629ee --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_slow.go @@ -0,0 +1,44 @@ +// Copyright 2024 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import "time" + +// SlowReader wraps a FrameReader and adds a fixed delay before each frame read, +// simulating a source that produces data slower than the pipeline can consume. +// This is useful for testing that the non-live pipeline correctly handles +// asymmetric source rates via backpressure rather than dropping data. +type SlowReader struct { + inner FrameReader + delay time.Duration +} + +// NewSlowReader wraps an existing FrameReader with a per-frame delay. +func NewSlowReader(inner FrameReader, delay time.Duration) *SlowReader { + return &SlowReader{inner: inner, delay: delay} +} + +func (r *SlowReader) NextFrame() ([]byte, uint32, error) { + time.Sleep(r.delay) + return r.inner.NextFrame() +} + +func (r *SlowReader) ClockRate() uint32 { + return r.inner.ClockRate() +} + +func (r *SlowReader) Close() error { + return r.inner.Close() +} diff --git a/pkg/pipeline/source/testfeeder/reader_stalling.go b/pkg/pipeline/source/testfeeder/reader_stalling.go new file mode 100644 index 000000000..cb6cc6c7a --- /dev/null +++ b/pkg/pipeline/source/testfeeder/reader_stalling.go @@ -0,0 +1,85 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testfeeder + +import "io" + +// StallingReader wraps a FrameReader and blocks forever after delivering a +// fixed number of frames. This simulates a track that has a gap in its +// recording — data stops arriving but no EOS or GAP event is sent. +type StallingReader struct { + inner FrameReader + maxFrames int + delivered int + stallCh chan struct{} // closed when the reader starts stalling + unblockCh chan struct{} // close this to unblock the stall (e.g. for cleanup) +} + +// NewStallingReader wraps an existing FrameReader. After maxFrames have been +// read, NextFrame blocks until unblockCh is closed (or forever if nil). +// stallCh is closed when the stall begins, allowing the test to detect it. +func NewStallingReader(inner FrameReader, maxFrames int) *StallingReader { + return &StallingReader{ + inner: inner, + maxFrames: maxFrames, + stallCh: make(chan struct{}), + unblockCh: make(chan struct{}), + } +} + +// Stalled returns a channel that is closed when the reader begins stalling. +func (r *StallingReader) Stalled() <-chan struct{} { + return r.stallCh +} + +// Unblock releases the stalled reader so the test can clean up. After +// unblocking, NextFrame returns io.EOF. +func (r *StallingReader) Unblock() { + select { + case <-r.unblockCh: + default: + close(r.unblockCh) + } +} + +func (r *StallingReader) NextFrame() ([]byte, uint32, error) { + if r.delivered >= r.maxFrames { + // Signal that we've started stalling. + select { + case <-r.stallCh: + default: + close(r.stallCh) + } + // Block until unblocked (test cleanup). + <-r.unblockCh + return nil, 0, io.EOF + } + + payload, samples, err := r.inner.NextFrame() + if err != nil { + return nil, 0, err + } + r.delivered++ + return payload, samples, nil +} + +func (r *StallingReader) ClockRate() uint32 { + return r.inner.ClockRate() +} + +func (r *StallingReader) Close() error { + r.Unblock() + return r.inner.Close() +} diff --git a/pkg/pipeline/source/testfeeder/source.go b/pkg/pipeline/source/testfeeder/source.go index a20fbaf1b..bf96d062b 100644 --- a/pkg/pipeline/source/testfeeder/source.go +++ b/pkg/pipeline/source/testfeeder/source.go @@ -82,6 +82,11 @@ type TrackDef struct { // TrackID is an optional identifier. Auto-generated if empty. TrackID string + + // Reader optionally overrides the file-based reader. When set, Path is + // ignored and this reader is used directly. This allows wrapping readers + // (e.g. SlowReader) for testing. + Reader FrameReader } // NewTestSource creates a TestSource that populates the PipelineConfig with @@ -101,10 +106,14 @@ func NewTestSource(conf *config.PipelineConfig, defs []TrackDef) (*TestSource, e def.TrackID = fmt.Sprintf("test_track_%d", i) } - reader, err := openReader(def) - if err != nil { - s.closeReaders() - return nil, fmt.Errorf("opening %s: %w", def.Path, err) + reader := def.Reader + if reader == nil { + var err error + reader, err = openReader(def) + if err != nil { + s.closeReaders() + return nil, fmt.Errorf("opening %s: %w", def.Path, err) + } } src, err := gst.NewElementWithName("appsrc", fmt.Sprintf("app_%s", def.TrackID)) diff --git a/pkg/pipeline/source/testfeeder/source_test.go b/pkg/pipeline/source/testfeeder/source_test.go index 5f25d25a0..c187f87f2 100644 --- a/pkg/pipeline/source/testfeeder/source_test.go +++ b/pkg/pipeline/source/testfeeder/source_test.go @@ -151,3 +151,418 @@ func TestAudioOnlyOGG(t *testing.T) { t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) } + +// TestAsymmetricAudioRates tests that the non-live pipeline correctly handles +// two audio sources producing at different rates. One track pushes at full speed +// while the other is throttled to roughly realtime (20ms sleep per 20ms Opus +// frame). The audiomixer should wait for the slow source via backpressure — no +// data loss, valid output. +func TestAsymmetricAudioRates(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-asymmetric.ogg" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) + conf.Latency.AudioMixerLatency = 2750 * time.Millisecond + + gst.Init(nil) + + // Open the slow reader manually so we can wrap it. + // 20ms delay per frame ≈ realtime playback speed, roughly 2x slower than + // the fast track which pushes as fast as the pipeline accepts. + // Using a different audio file so both tracks are audibly distinguishable + // in the output. + slowInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) + require.NoError(t, err) + slowReader := testfeeder.NewSlowReader(slowInner, 20*time.Millisecond) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + TrackID: "fast_track", + Path: filepath.Join(samples, "SolLevante.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + { + TrackID: "slow_track", + MimeType: types.MimeTypeOpus, + PayloadType: 111, + Reader: slowReader, + }, + }) + require.NoError(t, err) + + require.Len(t, conf.AudioTracks, 2) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + t.Logf("slow track (120s audio at realtime) gates the pipeline, got %v wall clock", elapsed) + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} + +// TestMixerStallsOnGap proves that the non-live audiomixer blocks forever when +// one track stops producing data without sending EOS or a GAP event. This +// demonstrates why the replay coordinator must detect gaps in recorded RTP +// data and inject GAP events. +func TestMixerStallsOnGap(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-stall.ogg" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) + conf.Latency.AudioMixerLatency = 2750 * time.Millisecond + + gst.Init(nil) + + // The stalling reader delivers 50 frames (~1s of Opus) then blocks forever, + // simulating a track with a gap in its recording. + stallingInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) + require.NoError(t, err) + stallingReader := testfeeder.NewStallingReader(stallingInner, 50) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + TrackID: "normal_track", + Path: filepath.Join(samples, "SolLevante.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + { + TrackID: "stalling_track", + MimeType: types.MimeTypeOpus, + PayloadType: 111, + Reader: stallingReader, + }, + }) + require.NoError(t, err) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + // Run the pipeline in a goroutine since it will stall. + done := make(chan *livekit.EgressInfo, 1) + go func() { + done <- ctrl.Run(ctx) + }() + + // Wait for the stalling reader to confirm it has stopped producing data. + select { + case <-stallingReader.Stalled(): + t.Log("stalling reader has stopped producing data after 50 frames") + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for stalling reader to stall") + } + + // The pipeline should NOT complete within 5 seconds — the mixer is blocked + // waiting for the stalling track. + select { + case info := <-done: + t.Fatalf("pipeline should have stalled but completed with status: %s", info.Status) + case <-time.After(5 * time.Second): + t.Log("CONFIRMED: pipeline is stalled — mixer is blocked waiting for missing data") + } + + // Clean up: unblock the stalling reader so the pipeline can shut down. + stallingReader.Unblock() + + select { + case info := <-done: + t.Logf("pipeline finished after unblock, status: %s", info.Status) + case <-time.After(30 * time.Second): + t.Fatal("pipeline did not finish after unblocking stalling reader") + } +} + +// TestGapEventUnblocksMixer proves that sending a GAP event on a track allows +// the audiomixer to proceed with silence instead of blocking forever. This is +// the fix for the stall demonstrated in TestMixerStallsOnGap. +// +// Track 1: full SolLevante.ogg (~28s), pushes at full speed. +// Track 2: 50 frames (~1s) of audio, then a 10s GAP event, then continues. +// +// The pipeline should complete quickly (faster than realtime) without stalling. +func TestGapEventUnblocksMixer(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-gap-event.ogg" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) + conf.Latency.AudioMixerLatency = 2750 * time.Millisecond + + gst.Init(nil) + + // After 50 frames (~1s), inject a 10s GAP event, then continue with + // remaining frames. The mixer should fill silence for track 2 during + // the gap and keep mixing track 1. + gappingInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) + require.NoError(t, err) + gappingReader := testfeeder.NewGappingReader(gappingInner, 50, 10*time.Second) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + TrackID: "normal_track", + Path: filepath.Join(samples, "SolLevante.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + { + TrackID: "gapping_track", + MimeType: types.MimeTypeOpus, + PayloadType: 111, + Reader: gappingReader, + }, + }) + require.NoError(t, err) + + require.Len(t, conf.AudioTracks, 2) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + // The pipeline should complete much faster than realtime — if it took + // >15s, something is wrong (the gap event didn't unblock the mixer). + require.Less(t, elapsed, 15*time.Second, + "pipeline should complete faster than realtime; GAP event may not have worked") + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} + +// TestRoomCompositeAudioOnly tests the non-live pipeline with two Opus audio +// tracks mixed through audiomixer, simulating an audio-only room composite with +// two participants. Both tracks are decoded, mixed, and re-encoded to OGG/Opus. +func TestRoomCompositeAudioOnly(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-room-audio.ogg" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) + conf.Latency.AudioMixerLatency = 2750 * time.Millisecond + + gst.Init(nil) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + TrackID: "participant_1_audio", + Path: filepath.Join(samples, "SolLevante.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + { + TrackID: "participant_2_audio", + Path: filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + }) + require.NoError(t, err) + + require.True(t, conf.AudioEnabled) + require.False(t, conf.VideoEnabled) + require.False(t, conf.Live) + require.Len(t, conf.AudioTracks, 2) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} + +// TestAudioVideoMP4 tests the non-live pipeline with both an Opus audio track +// and an H.264 video track producing an MP4 file. This simulates a participant +// egress: audio goes through Opus→decode→AAC encode, video passes through as +// H.264. +func TestAudioVideoMP4(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-av.mp4" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeMP4) + conf.VideoConfig = config.VideoConfig{ + VideoProfile: types.ProfileMain, + Width: 1920, + Height: 1080, + Depth: 24, + Framerate: 25, + VideoBitrate: 3000, + } + conf.AudioOutCodec = types.MimeTypeAAC + conf.AudioTranscoding = true + + gst.Init(nil) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + Path: filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg"), + MimeType: types.MimeTypeOpus, + PayloadType: 111, + }, + { + Path: filepath.Join(samples, "avsync_minmotion_livekit_video_1080p25_120s.h264"), + MimeType: types.MimeTypeH264, + PayloadType: 96, + FPS: 25, + }, + }) + require.NoError(t, err) + + require.True(t, conf.AudioEnabled) + require.True(t, conf.VideoEnabled) + require.False(t, conf.Live) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} + +// TestVideoOnlyH264MP4 tests the non-live pipeline with a single H.264 video +// track producing an MP4 file. No audio — just video appsrc → depay → mux → file. +func TestVideoOnlyH264MP4(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-video-h264.mp4" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeMP4) + conf.VideoConfig = config.VideoConfig{ + VideoProfile: types.ProfileMain, + Width: 1920, + Height: 1080, + Depth: 24, + Framerate: 25, + VideoBitrate: 3000, + } + + gst.Init(nil) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + Path: filepath.Join(samples, "avsync_minmotion_livekit_video_1080p25_120s.h264"), + MimeType: types.MimeTypeH264, + PayloadType: 96, + FPS: 25, + }, + }) + require.NoError(t, err) + + require.True(t, conf.VideoEnabled) + require.False(t, conf.Live) + require.NotNil(t, conf.VideoTrack) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} + +// TestVideoOnlyVP8WebM tests the non-live pipeline with a single VP8 video +// track producing a WebM file. +func TestVideoOnlyVP8WebM(t *testing.T) { + samples := mediaSamplesDir(t) + outputPath := "/tmp/testfeeder-video-vp8.webm" + + conf := newTestPipelineConfig(t, outputPath, types.OutputTypeWebM) + conf.VideoConfig = config.VideoConfig{ + VideoProfile: types.ProfileMain, + Width: 1920, + Height: 1080, + Depth: 24, + Framerate: 24, + VideoBitrate: 3000, + } + + gst.Init(nil) + + src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ + { + Path: filepath.Join(samples, "SolLevante-vp8.ivf"), + MimeType: types.MimeTypeVP8, + PayloadType: 96, + }, + }) + require.NoError(t, err) + + require.True(t, conf.VideoEnabled) + require.False(t, conf.Live) + require.NotNil(t, conf.VideoTrack) + + ctx := context.Background() + ctrl, err := pipeline.NewWithSource(ctx, conf, src) + require.NoError(t, err) + + src.SetCallbacks(ctrl.Callbacks()) + + start := time.Now() + info := ctrl.Run(ctx) + elapsed := time.Since(start) + + t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) + + stat, err := os.Stat(outputPath) + require.NoError(t, err, "output file should exist") + require.Greater(t, stat.Size(), int64(0), "output file should not be empty") + + t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) +} From ce2ad1916331ba13fb904f345ee10d32f9ac03bd Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Thu, 9 Apr 2026 10:53:59 +0200 Subject: [PATCH 3/9] Reject stream outputs for non live mode --- pkg/config/output.go | 10 + pkg/gstreamer/race_test.go | 459 ------------------------------------- 2 files changed, 10 insertions(+), 459 deletions(-) delete mode 100644 pkg/gstreamer/race_test.go diff --git a/pkg/config/output.go b/pkg/config/output.go index 579b5c9e4..b76d645ff 100644 --- a/pkg/config/output.go +++ b/pkg/config/output.go @@ -222,6 +222,16 @@ func (p *PipelineConfig) updateOutputs(req *livekit.ExportReplayRequest) error { return errors.ErrInvalidInput("output") } + // Non-live pipelines produce data faster than realtime. Stream outputs + // (RTMP, WebSocket) cannot ingest faster than 1x playback speed. + if !p.Live { + for _, output := range req.Outputs { + if _, ok := output.Config.(*livekit.Output_Stream); ok { + return errors.ErrNotSupported("stream output for non-live pipeline") + } + } + } + var hasFile, hasStream, hasSegments bool var fileCount, streamCount, segmentCount int diff --git a/pkg/gstreamer/race_test.go b/pkg/gstreamer/race_test.go deleted file mode 100644 index e8a1f3733..000000000 --- a/pkg/gstreamer/race_test.go +++ /dev/null @@ -1,459 +0,0 @@ -package gstreamer - -import ( - "fmt" - "math/rand" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/go-gst/go-glib/glib" - "github.com/go-gst/go-gst/gst" - "github.com/go-gst/go-gst/gst/app" - "github.com/stretchr/testify/require" -) - -var gstInitOnce sync.Once - -func initGStreamer(t *testing.T) { - t.Helper() - gstInitOnce.Do(func() { - gst.Init(nil) - }) -} - -// TestAddSourceBinRace attempts to reproduce the race condition where a source -// bin is added via AddSourceBin concurrently with the pipeline's PAUSED → PLAYING -// transition. In production this causes either FlowFlushing (priv->flushing stuck -// true) or a dead streaming thread (need-data never fires). -// -// Pipeline topology matches production: -// -// Source bins: appsrc → rtpopusdepay → opusdec → audiorate → queue(leaky) → audioconvert → audioresample → capsfilter -// Sink bin: audiotestsrc → capsfilter | audiomixer → capsfilter → queue → fakesink -// -// Production-matching timing: -// - GLib main loop running during the race (bus messages processed via watch, same as production) -// - Bus watch triggers concurrent PushBuffer (simulates pushSamples waking on Playing fuse) -// - Timing variants: bin-add-first (production pattern), simultaneous, playing-first -func TestAddSourceBinRace(t *testing.T) { - initGStreamer(t) - - const iterations = 1000 - - var ( - flowFlushing int - needDataMiss int - pushOK int - watchFired int - loopConfirm int - needDataOK int - errCount int - byVariant [3]int // [bin-add-first, simultaneous, playing-first] - ) - - for i := range iterations { - r := runRaceIteration(t, i) - if r.flowFlushing { - flowFlushing++ - } - if r.needDataMissing { - needDataMiss++ - } - if r.pushFlowOK { - pushOK++ - } - if r.watchTriggered { - watchFired++ - } - if r.loopRan { - loopConfirm++ - } - if r.needDataReceived { - needDataOK++ - } - if r.setupError { - errCount++ - } - byVariant[r.variant]++ - } - - t.Logf("results over %d iterations:", iterations) - t.Logf(" race hits: flowFlushing=%d needDataMiss=%d", flowFlushing, needDataMiss) - t.Logf(" healthy: loopConfirmed=%d watchFired=%d pushOK=%d needDataOK=%d", - loopConfirm, watchFired, pushOK, needDataOK) - t.Logf(" errors: setupErrors=%d", errCount) - t.Logf(" variants: binAddFirst=%d simultaneous=%d playingFirst=%d", - byVariant[0], byVariant[1], byVariant[2]) - - if flowFlushing > 0 || needDataMiss > 0 { - t.Logf("RACE DETECTED: reproduced the dynamic bin addition race condition") - } - - // Verify the test machinery actually worked on every non-error iteration - healthy := iterations - errCount - require.Equal(t, healthy, loopConfirm, "GLib main loop should have run on every non-error iteration") - require.Equal(t, healthy, watchFired, "bus watch should have fired on every non-error iteration") - require.Equal(t, healthy, pushOK+flowFlushing, "PushBuffer should have returned OK or FlowFlushing on every non-error iteration") - require.Equal(t, healthy, needDataOK+needDataMiss, "need-data should have been checked on every non-error iteration") -} - -const ( - raceAudioCaps = "audio/x-raw,format=S16LE,rate=48000,channels=2,layout=interleaved" - raceRTPCaps = "application/x-rtp,media=audio,payload=111,encoding-name=OPUS,clock-rate=48000" - // 3ms tolerance matches production audioRateTolerance - raceAudioRateTolerance = 3 * time.Millisecond - // 1s latency for queue max-size-time (production uses PipelineLatency) - raceQueueLatency = time.Second - - variantBinAddFirst = 0 - variantSimultaneous = 1 - variantPlayingFirst = 2 -) - -type raceResult struct { - flowFlushing bool - needDataMissing bool - pushFlowOK bool - watchTriggered bool - loopRan bool - needDataReceived bool - setupError bool - variant int -} - -// buildAudioConverterChain creates the production audio converter chain: -// audiorate → queue(leaky=downstream) → audioconvert → audioresample → capsfilter -func buildAudioConverterChain(t *testing.T) []*gst.Element { - t.Helper() - - audioRate, err := gst.NewElement("audiorate") - require.NoError(t, err) - require.NoError(t, audioRate.SetProperty("skip-to-first", true)) - require.NoError(t, audioRate.SetProperty("tolerance", uint64(raceAudioRateTolerance))) - - audioQueue, err := gst.NewElement("queue") - require.NoError(t, err) - require.NoError(t, audioQueue.SetProperty("max-size-time", uint64(raceQueueLatency))) - require.NoError(t, audioQueue.SetProperty("max-size-bytes", uint(0))) - require.NoError(t, audioQueue.SetProperty("max-size-buffers", uint(0))) - audioQueue.SetArg("leaky", "downstream") - - audioConvert, err := gst.NewElement("audioconvert") - require.NoError(t, err) - - audioResample, err := gst.NewElement("audioresample") - require.NoError(t, err) - - capsFilter, err := gst.NewElement("capsfilter") - require.NoError(t, err) - require.NoError(t, capsFilter.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) - - return []*gst.Element{audioRate, audioQueue, audioConvert, audioResample, capsFilter} -} - -func runRaceIteration(t *testing.T, iteration int) raceResult { - t.Helper() - - var result raceResult - - // Determine timing variant - switch v := iteration % 10; { - case v < 5: - result.variant = variantBinAddFirst - case v < 8: - result.variant = variantSimultaneous - default: - result.variant = variantPlayingFirst - } - - callbacks := &Callbacks{ - GstReady: make(chan struct{}), - BuildReady: make(chan struct{}), - } - close(callbacks.GstReady) - close(callbacks.BuildReady) - - p, err := NewPipeline("test_race", 0, callbacks) - require.NoError(t, err) - - // Build sink bin: audiotestsrc → capsfilter | audiomixer → capsfilter → queue → fakesink - audioBin := p.NewBin("audio") - - audioTestSrc, err := gst.NewElement("audiotestsrc") - require.NoError(t, err) - require.NoError(t, audioTestSrc.SetProperty("volume", 0.0)) - require.NoError(t, audioTestSrc.SetProperty("do-timestamp", true)) - require.NoError(t, audioTestSrc.SetProperty("is-live", true)) - require.NoError(t, audioTestSrc.SetProperty("samplesperbuffer", 960)) // 20ms @ 48kHz - - testSrcCaps, err := gst.NewElement("capsfilter") - require.NoError(t, err) - require.NoError(t, testSrcCaps.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) - - testSrcBin := p.NewBin("audio_test_src") - require.NoError(t, testSrcBin.AddElements(audioTestSrc, testSrcCaps)) - - audioMixer, err := gst.NewElement("audiomixer") - require.NoError(t, err) - - mixerCaps, err := gst.NewElement("capsfilter") - require.NoError(t, err) - require.NoError(t, mixerCaps.SetProperty("caps", gst.NewCapsFromString(raceAudioCaps))) - - outQueue, err := gst.NewElement("queue") - require.NoError(t, err) - - fakeSink, err := gst.NewElement("fakesink") - require.NoError(t, err) - require.NoError(t, fakeSink.SetProperty("async", false)) - - require.NoError(t, audioBin.AddElements(audioMixer, mixerCaps, outQueue, fakeSink)) - require.NoError(t, audioBin.AddSourceBin(testSrcBin)) - require.NoError(t, p.AddSinkBin(audioBin)) - - // Pre-existing source bins with full production chain to widen race window - const preExistingSources = 5 - preAppSrcs := make([]*app.Source, preExistingSources) - for i := range preExistingSources { - src, err := app.NewAppSrc() - require.NoError(t, err) - src.SetArg("format", "time") - require.NoError(t, src.SetProperty("is-live", true)) - require.NoError(t, src.SetProperty("caps", gst.NewCapsFromString(raceRTPCaps))) - - rtpDepay, err := gst.NewElement("rtpopusdepay") - require.NoError(t, err) - opusDec, err := gst.NewElement("opusdec") - require.NoError(t, err) - - srcBin := p.NewBin(fmt.Sprintf("app_pre_%d", i)) - srcBin.SetEOSFunc(func() bool { return false }) - require.NoError(t, srcBin.AddElement(src.Element)) - require.NoError(t, srcBin.AddElements(rtpDepay, opusDec)) - require.NoError(t, srcBin.AddElements(buildAudioConverterChain(t)...)) - require.NoError(t, audioBin.AddSourceBin(srcBin)) - preAppSrcs[i] = src - } - - require.NoError(t, p.Link()) - - _, ok := p.UpgradeState(StateStarted) - require.True(t, ok) - - require.NoError(t, p.SetState(gst.StatePaused)) - - // Prepare the race source bin with full production chain: - // appsrc → rtpopusdepay → opusdec → audiorate → queue(leaky) → audioconvert → audioresample → capsfilter - raceSrc, err := app.NewAppSrc() - require.NoError(t, err) - raceSrc.SetArg("format", "time") - require.NoError(t, raceSrc.SetProperty("is-live", true)) - require.NoError(t, raceSrc.SetProperty("caps", gst.NewCapsFromString(raceRTPCaps))) - - rtpOpusDepay, err := gst.NewElement("rtpopusdepay") - require.NoError(t, err) - - opusDec, err := gst.NewElement("opusdec") - require.NoError(t, err) - - raceBin := p.NewBin("app_race_track") - raceBin.SetEOSFunc(func() bool { return false }) - require.NoError(t, raceBin.AddElement(raceSrc.Element)) - require.NoError(t, raceBin.AddElements(rtpOpusDepay, opusDec)) - require.NoError(t, raceBin.AddElements(buildAudioConverterChain(t)...)) - - // Install need-data callback BEFORE the race so we don't miss the first emission. - needData := make(chan struct{}, 1) - raceSrc.SetCallbacks(&app.SourceCallbacks{ - NeedDataFunc: func(_ *app.Source, _ uint) { - select { - case needData <- struct{}{}: - default: - } - }, - }) - - // Install a bus watch (processed by the GLib main loop, same as production). - // When the race bin reports PLAYING, break a "fuse" (channel close) so a - // separate pushSamples goroutine wakes up — matching the production path: - // bus watch → Playing(trackID) → opChan → track worker → writer.Playing() - // → fuse breaks → pushSamples goroutine wakes on separate OS thread - // Return false once done to remove the GSource from the default context. - var watchDone atomic.Bool - playingFuse := make(chan struct{}) - p.SetWatch(func(msg *gst.Message) bool { - if watchDone.Load() { - return false // remove watch - } - if msg.Type() == gst.MessageStateChanged { - _, newState := msg.ParseStateChanged() - if newState == gst.StatePlaying && strings.HasPrefix(msg.Source(), "app_race") { - watchDone.Store(true) - close(playingFuse) // break the fuse - return false // remove watch - } - } - return true - }) - - // Simulate pushSamples: a separate goroutine (own OS thread) waits for the - // playing fuse, then pushes buffers in a tight loop — matching production - // where pushSamples continuously dequeues from the jitter buffer and calls - // PushBuffer. This catches narrow flushing windows that a single push misses. - const pushCount = 50 - pushResult := make(chan gst.FlowReturn, 1) - go func() { - runtime.LockOSThread() - select { - case <-playingFuse: - case <-time.After(2 * time.Second): - pushResult <- gst.FlowError - return - } - for i := range pushCount { - buf := gst.NewBufferWithSize(96) - pts := gst.ClockTime(uint64(i) * uint64(time.Millisecond)) - buf.SetPresentationTimestamp(pts) - buf.SetDuration(gst.ClockTime(time.Millisecond)) - if flow := raceSrc.PushBuffer(buf); flow != gst.FlowOK { - pushResult <- flow - return - } - } - pushResult <- gst.FlowOK - }() - - // Start the GLib main loop (same as production's Pipeline.Run()). - // Use IdleAdd to confirm the loop is actually iterating before proceeding. - loopRunning := make(chan struct{}) - loopDone := make(chan struct{}) - glib.IdleAdd(func() bool { - select { - case <-loopRunning: - default: - close(loopRunning) - } - return false // remove idle source - }) - go func() { - runtime.LockOSThread() - p.loop.Run() - close(loopDone) - }() - - select { - case <-loopRunning: - result.loopRan = true - case <-time.After(time.Second): - t.Fatal("GLib main loop did not start within 1s") - } - - // Race: SetState(PLAYING) vs AddSourceBin - playingDone := make(chan error, 1) - addDone := make(chan error, 1) - - switch result.variant { - case variantBinAddFirst: - // Bin-add-first with 0-100μs head start - // (matches production: "track subscribes ~1ms before PAUSED→PLAYING") - go func() { - runtime.LockOSThread() - addDone <- audioBin.AddSourceBin(raceBin) - }() - time.Sleep(time.Duration(rand.Int63n(100)) * time.Microsecond) - go func() { - runtime.LockOSThread() - playingDone <- p.SetState(gst.StatePlaying) - }() - - case variantSimultaneous: - // Simultaneous: barrier-synchronized start - barrier := make(chan struct{}) - go func() { - runtime.LockOSThread() - <-barrier - playingDone <- p.SetState(gst.StatePlaying) - }() - go func() { - runtime.LockOSThread() - <-barrier - addDone <- audioBin.AddSourceBin(raceBin) - }() - close(barrier) - - case variantPlayingFirst: - // Playing-first with 0-100μs head start - go func() { - runtime.LockOSThread() - playingDone <- p.SetState(gst.StatePlaying) - }() - time.Sleep(time.Duration(rand.Int63n(100)) * time.Microsecond) - go func() { - runtime.LockOSThread() - addDone <- audioBin.AddSourceBin(raceBin) - }() - } - - err1 := <-playingDone - err2 := <-addDone - - if err1 != nil || err2 != nil { - result.setupError = true - watchDone.Store(true) - p.loop.Quit() - <-loopDone - _ = p.SetState(gst.StateNull) - return result - } - - // Wait for pushSamples goroutine result (fuse break + 50 buffer pushes) - select { - case flow := <-pushResult: - result.watchTriggered = true // fuse broke (bus watch fired) - switch flow { - case gst.FlowFlushing: - result.flowFlushing = true - case gst.FlowOK: - result.pushFlowOK = true // all 50 pushes succeeded - } - case <-time.After(2 * time.Second): - // Bus watch never saw PLAYING for the race bin - result.needDataMissing = true - } - - // Check need-data — if the streaming thread is alive, it should have - // fired by now (or shortly after PushBuffer woke it from cond_wait). - if !result.flowFlushing && !result.needDataMissing { - select { - case <-needData: - result.needDataReceived = true - case <-time.After(500 * time.Millisecond): - result.needDataMissing = true - } - } - - if result.flowFlushing || result.needDataMissing { - t.Logf("--- RACE HIT iter=%d (flowFlushing=%v, needDataMissing=%v) ---", - iteration, result.flowFlushing, result.needDataMissing) - t.Logf("appsrc state: %s", raceSrc.Element.GetCurrentState().String()) - t.Logf("race bin state: %s", raceBin.bin.GetCurrentState().String()) - t.Logf("pipeline state: %s", p.pipeline.GetCurrentState().String()) - } - - // Cleanup: quit loop first, then destroy pipeline - watchDone.Store(true) - p.loop.Quit() - <-loopDone - _ = raceSrc.EndStream() - for _, src := range preAppSrcs { - _ = src.EndStream() - } - _ = p.SetState(gst.StateNull) - time.Sleep(10 * time.Millisecond) - - return result -} From 062aab7102dc9b845a86933188dbbd4a0aba3728 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Thu, 9 Apr 2026 15:55:07 +0200 Subject: [PATCH 4/9] Use live vs replay for pipeline terminology - for replay integration add a bool method, linter fixes --- pkg/config/pipeline.go | 7 +++++++ pkg/handler/handler.go | 2 +- pkg/pipeline/controller.go | 4 ++++ pkg/pipeline/source/testfeeder/feeder.go | 2 +- pkg/pipeline/source/testfeeder/reader_gapping.go | 12 ++++++------ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index 1d007d561..84d548c5c 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -62,6 +62,13 @@ type PipelineConfig struct { Live bool `yaml:"-"` } +// IsReplay returns true when this is a replay/export pipeline. Use this for +// replay-specific integration points (IPC calls, storage access). For generic +// pipeline behavior (is-live, leaky queues, backpressure) use the Live field. +func (p *PipelineConfig) IsReplay() bool { + return !p.Live +} + var ( tracer = otel.Tracer("github.com/livekit/egress/pkg/config") ) diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 4c370dff9..4d81e63c9 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -133,7 +133,7 @@ func (h *Handler) Run() { } // Replay coordination: signal ready and get timing - if !h.conf.Live { + if h.conf.IsReplay() { rctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) resp, err := h.ipcServiceClient.ReplayReady(rctx, &rpc.EgressReadyRequest{ EgressId: h.conf.Info.EgressId, diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 1f0be6a1d..bae2656ce 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -137,11 +137,15 @@ func New(ctx context.Context, conf *config.PipelineConfig, ipcServiceClient ipc. // testing, or ReplaySource for offline export). The source must have already // populated the PipelineConfig with track information before calling this. func NewWithSource(ctx context.Context, conf *config.PipelineConfig, src source.Source) (*Controller, error) { + ctx, span := tracer.Start(ctx, "Pipeline.NewWithSource") + defer span.End() c := newController(conf, nil) c.src = src // initialize gst go func() { + _, span := tracer.Start(ctx, "gst.Init") + defer span.End() gst.Init(nil) gst.SetLogFunction(c.gstLog) close(c.callbacks.GstReady) diff --git a/pkg/pipeline/source/testfeeder/feeder.go b/pkg/pipeline/source/testfeeder/feeder.go index 2edd27715..080491cc7 100644 --- a/pkg/pipeline/source/testfeeder/feeder.go +++ b/pkg/pipeline/source/testfeeder/feeder.go @@ -156,7 +156,7 @@ func (f *TrackFeeder) Feed() error { ) return nil } - var gapErr *ErrGap + var gapErr *GapError if errors.As(err, &gapErr) { pts := rtpToDuration(f.currentTS, clockRate) logger.Infow("feeder sending GAP event", diff --git a/pkg/pipeline/source/testfeeder/reader_gapping.go b/pkg/pipeline/source/testfeeder/reader_gapping.go index 786219fb9..7d3f8513e 100644 --- a/pkg/pipeline/source/testfeeder/reader_gapping.go +++ b/pkg/pipeline/source/testfeeder/reader_gapping.go @@ -20,19 +20,19 @@ import ( "time" ) -// ErrGap is returned by a FrameReader to signal a gap in the media data. +// GapError is returned by a FrameReader to signal a gap in the media data. // The feeder should send a GStreamer GAP event for the specified duration // instead of pushing a buffer, then continue reading. -type ErrGap struct { +type GapError struct { Duration time.Duration } -func (e *ErrGap) Error() string { +func (e *GapError) Error() string { return fmt.Sprintf("gap: %v", e.Duration) } // GappingReader wraps a FrameReader and introduces a gap after a fixed number -// of frames. After delivering gapAfter frames, it returns an ErrGap with the +// of frames. After delivering gapAfter frames, it returns an GapError with the // specified duration. After the gap, it continues reading from the inner reader // until EOF. type GappingReader struct { @@ -44,7 +44,7 @@ type GappingReader struct { } // NewGappingReader wraps an existing FrameReader. After gapAfter frames, the -// next call to NextFrame returns ErrGap with the given duration. Subsequent +// next call to NextFrame returns GapError with the given duration. Subsequent // calls continue reading from inner. func NewGappingReader(inner FrameReader, gapAfter int, gapDuration time.Duration) *GappingReader { return &GappingReader{ @@ -57,7 +57,7 @@ func NewGappingReader(inner FrameReader, gapAfter int, gapDuration time.Duration func (r *GappingReader) NextFrame() ([]byte, uint32, error) { if r.delivered >= r.gapAfter && !r.gapSent { r.gapSent = true - return nil, 0, &ErrGap{Duration: r.gapDuration} + return nil, 0, &GapError{Duration: r.gapDuration} } payload, samples, err := r.inner.NextFrame() From ccc7c4f46478cf8ce7678027a69c7fb5257b2717 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Thu, 9 Apr 2026 16:05:11 +0200 Subject: [PATCH 5/9] removing debug probes --- pkg/pipeline/builder/audio.go | 121 ---------------------------------- 1 file changed, 121 deletions(-) diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 24f366ea0..b0ef0f832 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -482,126 +482,9 @@ func (b *AudioBin) addMixer() error { subscribeForQoS(audioMixer) - if !b.conf.Live { - installMixerProbes(audioMixer) - } - return b.bin.AddElements(audioMixer, mixedCaps) } -// installTestSrcDisconnect sets up a probe on the mixer that sends EOS to the -// audiotestsrc once the first real audio buffer arrives from our appsrc path. -// The audiotestsrc is only needed for preroll — once real data is flowing, it -func installMixerProbes(mixer *gst.Element) { - // Install probes on sink pads as they're created (request pads) - mixer.Connect("pad-added", func(_ *gst.Element, pad *gst.Pad) { - name := pad.GetName() - var count uint64 - pad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - count++ - buf := info.GetBuffer() - if buf != nil && (count <= 5 || count%100 == 0) { - logger.Infow("mixer sink probe", - "pad", name, - "count", count, - "pts", buf.PresentationTimestamp(), - "dur", buf.Duration(), - "size", buf.GetSize(), - ) - } - return gst.PadProbeOK - }) - pad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - if event := info.GetEvent(); event != nil { - logger.Infow("mixer sink event", "pad", name, "type", event.Type().String()) - } - return gst.PadProbeOK - }) - logger.Infow("mixer probe installed", "pad", name) - }) - - // Src pad is static — probe it directly - srcPad := mixer.GetStaticPad("src") - if srcPad != nil { - var srcCount uint64 - srcPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - srcCount++ - buf := info.GetBuffer() - if buf != nil && (srcCount <= 5 || srcCount%100 == 0) { - logger.Infow("mixer src probe", - "count", srcCount, - "pts", buf.PresentationTimestamp(), - "dur", buf.Duration(), - "size", buf.GetSize(), - ) - } - return gst.PadProbeOK - }) - srcPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - if event := info.GetEvent(); event != nil { - logger.Infow("mixer src event", "type", event.Type().String()) - } - return gst.PadProbeOK - }) - } -} - -func installQueueProbes(queue *gst.Element, name string) { - sinkPad := queue.GetStaticPad("sink") - if sinkPad != nil { - var sinkBufs uint64 - sinkPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - sinkBufs++ - buf := info.GetBuffer() - if buf != nil { - curTime, _ := queue.GetProperty("current-level-time") - curBufs, _ := queue.GetProperty("current-level-buffers") - logger.Infow("queue sink buf", - "queue", name, - "count", sinkBufs, - "pts", buf.PresentationTimestamp(), - "dur", buf.Duration(), - "size", buf.GetSize(), - "qTime", curTime, - "qBufs", curBufs, - ) - } - return gst.PadProbeOK - }) - sinkPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - if event := info.GetEvent(); event != nil { - logger.Infow("queue sink event", "queue", name, "type", event.Type().String(), "bufsSoFar", sinkBufs) - } - return gst.PadProbeOK - }) - } - - srcPad := queue.GetStaticPad("src") - if srcPad != nil { - var srcBufs uint64 - srcPad.AddProbe(gst.PadProbeTypeBuffer, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - srcBufs++ - buf := info.GetBuffer() - if buf != nil && (srcBufs <= 3 || srcBufs%100 == 0) { - logger.Infow("queue src buf", - "queue", name, - "count", srcBufs, - "pts", buf.PresentationTimestamp(), - "dur", buf.Duration(), - "size", buf.GetSize(), - ) - } - return gst.PadProbeOK - }) - srcPad.AddProbe(gst.PadProbeTypeEventDownstream, func(_ *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - if event := info.GetEvent(); event != nil { - logger.Infow("queue src event", "queue", name, "type", event.Type().String(), "srcBufsSoFar", srcBufs) - } - return gst.PadProbeOK - }) - } -} - func (b *AudioBin) addEncoder() error { switch b.conf.AudioOutCodec { case types.MimeTypeOpus: @@ -656,10 +539,6 @@ func addAudioConverter(b *gstreamer.Bin, p *config.PipelineConfig, channel livek return err } - if !p.Live { - installQueueProbes(audioQueue, "audio_input_queue") - } - audioConvert, err := gst.NewElement("audioconvert") if err != nil { return errors.ErrGstPipelineError(err) From 74664df5cfc01a5aba2a78039e61abf9c9c4ad36 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Thu, 9 Apr 2026 16:16:43 +0200 Subject: [PATCH 6/9] terminology fixes --- pkg/pipeline/source/testfeeder/feeder.go | 2 +- pkg/pipeline/source/testfeeder/source_test.go | 33 ++++++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/pkg/pipeline/source/testfeeder/feeder.go b/pkg/pipeline/source/testfeeder/feeder.go index 080491cc7..3d6539496 100644 --- a/pkg/pipeline/source/testfeeder/feeder.go +++ b/pkg/pipeline/source/testfeeder/feeder.go @@ -17,7 +17,7 @@ // OGG), packetizes the frames into RTP packets, and pushes them to an appsrc as // fast as the pipeline will accept. This is used to test the egress GStreamer // pipeline in non-live (faster-than-realtime) mode without needing a live -// WebRTC source or the full replay infrastructure. +// WebRTC source. package testfeeder import ( diff --git a/pkg/pipeline/source/testfeeder/source_test.go b/pkg/pipeline/source/testfeeder/source_test.go index c187f87f2..324f10f83 100644 --- a/pkg/pipeline/source/testfeeder/source_test.go +++ b/pkg/pipeline/source/testfeeder/source_test.go @@ -42,19 +42,34 @@ func init() { // mediaSamplesDir returns the path to the media-samples directory. // Set MEDIA_SAMPLES_DIR env var to override, otherwise defaults to the -// cloud-egress media-samples relative to this repo. +// media-samples directory at the repo root. func mediaSamplesDir(t *testing.T) string { if dir := os.Getenv("MEDIA_SAMPLES_DIR"); dir != "" { return dir } - // Default: assume egress and cloud-egress are sibling dirs - dir := filepath.Join("..", "..", "..", "..", "..", "cloud-egress", "media-samples") - abs, err := filepath.Abs(dir) + dir := repoRoot(t) + samples := filepath.Join(dir, "media-samples") + if _, err := os.Stat(samples); err != nil { + t.Skipf("media-samples not found at %s (set MEDIA_SAMPLES_DIR)", samples) + } + return samples +} + +// repoRoot walks up from the current file's directory until it finds go.mod. +func repoRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() require.NoError(t, err) - if _, err := os.Stat(abs); err != nil { - t.Skipf("media-samples not found at %s (set MEDIA_SAMPLES_DIR)", abs) + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("could not find repo root (no go.mod found)") + } + dir = parent } - return abs } // newTestPipelineConfig creates a minimal PipelineConfig suitable for pipeline @@ -215,8 +230,8 @@ func TestAsymmetricAudioRates(t *testing.T) { // TestMixerStallsOnGap proves that the non-live audiomixer blocks forever when // one track stops producing data without sending EOS or a GAP event. This -// demonstrates why the replay coordinator must detect gaps in recorded RTP -// data and inject GAP events. +// demonstrates why non-live sources must detect gaps in their input data and +// inject GAP events. func TestMixerStallsOnGap(t *testing.T) { samples := mediaSamplesDir(t) outputPath := "/tmp/testfeeder-stall.ogg" From 236d141d340ef3b9dc1f92e84dc99782f00b56ae Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Thu, 9 Apr 2026 16:51:59 +0200 Subject: [PATCH 7/9] remove stream check --- pkg/config/pipeline.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index 84d548c5c..23c7e1283 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -426,7 +426,6 @@ func (p *PipelineConfig) Update(request *rpc.StartEgressRequest) error { } case *rpc.StartEgressRequest_Replay: - p.Live = false replayReq := req.Replay clone := proto.Clone(replayReq).(*livekit.ExportReplayRequest) p.Info.Request = &livekit.EgressInfo_Replay{ From 39a5702c42f092533cb4b30bcb392e64ac11ac69 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Fri, 17 Apr 2026 14:29:10 +0200 Subject: [PATCH 8/9] Removing test feeder --- pkg/pipeline/source/testfeeder/feeder.go | 272 -------- .../source/testfeeder/reader_gapping.go | 80 --- pkg/pipeline/source/testfeeder/reader_h264.go | 81 --- pkg/pipeline/source/testfeeder/reader_ivf.go | 93 --- pkg/pipeline/source/testfeeder/reader_ogg.go | 148 ----- pkg/pipeline/source/testfeeder/reader_slow.go | 44 -- .../source/testfeeder/reader_stalling.go | 85 --- pkg/pipeline/source/testfeeder/source.go | 281 --------- pkg/pipeline/source/testfeeder/source_test.go | 583 ------------------ 9 files changed, 1667 deletions(-) delete mode 100644 pkg/pipeline/source/testfeeder/feeder.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_gapping.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_h264.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_ivf.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_ogg.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_slow.go delete mode 100644 pkg/pipeline/source/testfeeder/reader_stalling.go delete mode 100644 pkg/pipeline/source/testfeeder/source.go delete mode 100644 pkg/pipeline/source/testfeeder/source_test.go diff --git a/pkg/pipeline/source/testfeeder/feeder.go b/pkg/pipeline/source/testfeeder/feeder.go deleted file mode 100644 index 3d6539496..000000000 --- a/pkg/pipeline/source/testfeeder/feeder.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package testfeeder provides a non-live media file feeder for GStreamer appsrc -// elements. It reads encoded media from files (H.264 annexb, VP8/VP9 IVF, Opus -// OGG), packetizes the frames into RTP packets, and pushes them to an appsrc as -// fast as the pipeline will accept. This is used to test the egress GStreamer -// pipeline in non-live (faster-than-realtime) mode without needing a live -// WebRTC source. -package testfeeder - -import ( - "errors" - "fmt" - "io" - "time" - - "github.com/go-gst/go-gst/gst" - "github.com/go-gst/go-gst/gst/app" - "github.com/pion/rtp" - "github.com/pion/rtp/codecs" - - "github.com/livekit/egress/pkg/types" - "github.com/livekit/protocol/logger" -) - -const ( - defaultMTU = 1200 - videoClockRate = 90000 - opusClockRate = 48000 - defaultVideoFPS = 30 - defaultOpusFrame = 20 * time.Millisecond -) - -// FrameReader reads encoded media frames from a file. -// Each call to NextFrame returns the raw encoded data for one frame -// and the frame's duration in RTP clock rate units (samples). -type FrameReader interface { - // NextFrame returns the next frame's payload and its duration in RTP - // timestamp units (samples at the codec clock rate). Returns io.EOF - // when all frames have been read. - NextFrame() (payload []byte, samples uint32, err error) - - // ClockRate returns the RTP clock rate for this media type. - ClockRate() uint32 - - io.Closer -} - -// TrackFeeder reads frames from a FrameReader, packetizes them into RTP, and -// pushes the resulting buffers to a GStreamer appsrc element. When the appsrc -// is configured with is-live=false, PushBuffer blocks on backpressure, -// naturally throttling the feeder to the pipeline's processing speed. -type TrackFeeder struct { - src *app.Source - reader FrameReader - packetizer rtp.Packetizer - codec types.MimeType - startTS uint32 - currentTS uint32 - ptsOffset time.Duration // accumulated offset from GAP events -} - -// TrackFeederParams configures a TrackFeeder. -type TrackFeederParams struct { - // AppSrc is the GStreamer appsrc element to push buffers to. - AppSrc *app.Source - - // MimeType selects the codec (types.MimeTypeH264, MimeTypeVP8, MimeTypeOpus, etc.). - MimeType types.MimeType - - // PayloadType is the RTP payload type number (e.g. 96 for H264, 111 for Opus). - PayloadType uint8 - - // SSRC for the RTP stream. Use any fixed value for testing. - SSRC uint32 - - // Reader is the frame source. Use NewH264Reader, NewIVFReader, or NewOGGReader. - Reader FrameReader -} - -// NewTrackFeeder creates a TrackFeeder that reads from the given FrameReader -// and pushes RTP-packetized buffers to the appsrc. -func NewTrackFeeder(params TrackFeederParams) (*TrackFeeder, error) { - payloader, err := payloaderForCodec(params.MimeType) - if err != nil { - return nil, err - } - - clockRate := params.Reader.ClockRate() - - packetizer := rtp.NewPacketizer( - defaultMTU, - params.PayloadType, - params.SSRC, - payloader, - rtp.NewRandomSequencer(), - clockRate, - ) - - return &TrackFeeder{ - src: params.AppSrc, - reader: params.Reader, - packetizer: packetizer, - codec: params.MimeType, - }, nil -} - -// Feed reads all frames from the reader, packetizes them, and pushes them to -// the appsrc. It blocks until all frames are consumed (io.EOF from reader) or -// an error occurs. On completion it sends an EOS event to the appsrc. -// -// With is-live=false on the appsrc, PushBuffer provides natural backpressure: -// when the downstream pipeline is busy, the push blocks, so no rate-limiting -// code is needed. -func (f *TrackFeeder) Feed() error { - defer f.sendEOS() - - // Log appsrc properties - if maxBuf, err := f.src.GetProperty("max-buffers"); err == nil { - logger.Infow("appsrc config", - "max-buffers", maxBuf, - "is-live", func() interface{} { v, _ := f.src.GetProperty("is-live"); return v }(), - "max-bytes", func() interface{} { v, _ := f.src.GetProperty("max-bytes"); return v }(), - "current-level-buffers", func() interface{} { v, _ := f.src.GetProperty("current-level-buffers"); return v }(), - ) - } - - clockRate := f.reader.ClockRate() - firstFrame := true - frameCount := 0 - packetCount := 0 - totalBytes := 0 - start := time.Now() - - for { - payload, samples, err := f.reader.NextFrame() - if err == io.EOF { - logger.Infow("feeder done", - "frames", frameCount, - "packets", packetCount, - "bytes", totalBytes, - "elapsed", time.Since(start), - "codec", f.codec, - ) - return nil - } - var gapErr *GapError - if errors.As(err, &gapErr) { - pts := rtpToDuration(f.currentTS, clockRate) - logger.Infow("feeder sending GAP event", - "pts", pts, - "gapDuration", gapErr.Duration, - "framesBeforeGap", frameCount, - ) - if err := f.sendGapEvent(pts+f.ptsOffset, gapErr.Duration); err != nil { - return fmt.Errorf("sending GAP event: %w", err) - } - // Accumulate the gap so subsequent buffer PTS values are shifted forward. - f.ptsOffset += gapErr.Duration - continue - } - if err != nil { - return fmt.Errorf("reading frame: %w", err) - } - - frameCount++ - packets := f.packetizer.Packetize(payload, samples) - for _, pkt := range packets { - if firstFrame { - // Capture the actual RTP timestamp from the first packet. - // The packetizer starts with a random base timestamp. - f.startTS = pkt.Timestamp - firstFrame = false - } - packetCount++ - if err := f.pushRTPPacket(pkt, clockRate); err != nil { - return fmt.Errorf("pushing packet seq=%d: %w", pkt.SequenceNumber, err) - } - totalBytes += len(pkt.Payload) - } - - f.currentTS += samples - - if frameCount%100 == 0 { - pts := rtpToDuration(f.currentTS, clockRate) - logger.Debugw("feeder progress", - "frames", frameCount, - "packets", packetCount, - "mediaPTS", pts, - "elapsed", time.Since(start), - ) - } - } -} - -// pushRTPPacket marshals an RTP packet, creates a GstBuffer with the -// appropriate PTS, and pushes it to the appsrc. -func (f *TrackFeeder) pushRTPPacket(pkt *rtp.Packet, clockRate uint32) error { - buf, err := pkt.Marshal() - if err != nil { - return fmt.Errorf("marshaling RTP packet: %w", err) - } - - pts := rtpToDuration(pkt.Timestamp-f.startTS, clockRate) + f.ptsOffset - - b := gst.NewBufferFromBytes(buf) - b.SetPresentationTimestamp(gst.ClockTime(uint64(pts))) - - flow := f.src.PushBuffer(b) - if flow != gst.FlowOK { - return fmt.Errorf("appsrc push returned %v", flow) - } - return nil -} - -func (f *TrackFeeder) sendEOS() { - f.src.EndStream() -} - -// sendGapEvent pushes a GAP event downstream from the appsrc, telling the -// mixer to insert silence for this track over the given duration. -// -// GAP is a downstream serialized event. We push it via Pad.PushEvent on the -// appsrc's src pad (not SendEvent, which is for upstream events). -func (f *TrackFeeder) sendGapEvent(pts time.Duration, duration time.Duration) error { - event := gst.NewGapEvent( - gst.ClockTime(uint64(pts)), - gst.ClockTime(uint64(duration)), - ) - srcPad := f.src.GetStaticPad("src") - if srcPad == nil { - return fmt.Errorf("appsrc has no src pad") - } - if !srcPad.PushEvent(event) { - return fmt.Errorf("failed to push GAP event on appsrc src pad") - } - return nil -} - -// rtpToDuration converts an RTP timestamp delta to a time.Duration. -func rtpToDuration(rtpTS uint32, clockRate uint32) time.Duration { - return time.Duration(float64(rtpTS) / float64(clockRate) * float64(time.Second)) -} - -func payloaderForCodec(mime types.MimeType) (rtp.Payloader, error) { - switch mime { - case types.MimeTypeH264: - return &codecs.H264Payloader{}, nil - case types.MimeTypeVP8: - return &codecs.VP8Payloader{ - EnablePictureID: true, - }, nil - case types.MimeTypeVP9: - return &codecs.VP9Payloader{}, nil - case types.MimeTypeOpus: - return &codecs.OpusPayloader{}, nil - default: - return nil, fmt.Errorf("unsupported codec for test feeder: %s", mime) - } -} diff --git a/pkg/pipeline/source/testfeeder/reader_gapping.go b/pkg/pipeline/source/testfeeder/reader_gapping.go deleted file mode 100644 index 7d3f8513e..000000000 --- a/pkg/pipeline/source/testfeeder/reader_gapping.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import ( - "fmt" - "io" - "time" -) - -// GapError is returned by a FrameReader to signal a gap in the media data. -// The feeder should send a GStreamer GAP event for the specified duration -// instead of pushing a buffer, then continue reading. -type GapError struct { - Duration time.Duration -} - -func (e *GapError) Error() string { - return fmt.Sprintf("gap: %v", e.Duration) -} - -// GappingReader wraps a FrameReader and introduces a gap after a fixed number -// of frames. After delivering gapAfter frames, it returns an GapError with the -// specified duration. After the gap, it continues reading from the inner reader -// until EOF. -type GappingReader struct { - inner FrameReader - gapAfter int - gapDuration time.Duration - delivered int - gapSent bool -} - -// NewGappingReader wraps an existing FrameReader. After gapAfter frames, the -// next call to NextFrame returns GapError with the given duration. Subsequent -// calls continue reading from inner. -func NewGappingReader(inner FrameReader, gapAfter int, gapDuration time.Duration) *GappingReader { - return &GappingReader{ - inner: inner, - gapAfter: gapAfter, - gapDuration: gapDuration, - } -} - -func (r *GappingReader) NextFrame() ([]byte, uint32, error) { - if r.delivered >= r.gapAfter && !r.gapSent { - r.gapSent = true - return nil, 0, &GapError{Duration: r.gapDuration} - } - - payload, samples, err := r.inner.NextFrame() - if err != nil { - if err == io.EOF { - return nil, 0, io.EOF - } - return nil, 0, err - } - r.delivered++ - return payload, samples, nil -} - -func (r *GappingReader) ClockRate() uint32 { - return r.inner.ClockRate() -} - -func (r *GappingReader) Close() error { - return r.inner.Close() -} diff --git a/pkg/pipeline/source/testfeeder/reader_h264.go b/pkg/pipeline/source/testfeeder/reader_h264.go deleted file mode 100644 index a4041a0b7..000000000 --- a/pkg/pipeline/source/testfeeder/reader_h264.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import ( - "io" - "os" - - "github.com/pion/webrtc/v4/pkg/media/h264reader" -) - -// H264Reader reads H.264 NAL units from an Annex B bitstream file and presents -// them as frames suitable for RTP packetization. -// -// Each call to NextFrame returns a complete NAL unit. The frame duration is -// fixed at (clockRate / fps) samples, which is correct for constant-framerate -// content. For VFR content the duration is approximate, but sufficient for -// pipeline testing. -type H264Reader struct { - reader *h264reader.H264Reader - file *os.File - fps int -} - -// NewH264Reader opens an H.264 Annex B file and returns a FrameReader. -// fps sets the assumed frame rate for timestamp advancement. -func NewH264Reader(path string, fps int) (*H264Reader, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - h, err := h264reader.NewReader(f) - if err != nil { - f.Close() - return nil, err - } - - if fps <= 0 { - fps = defaultVideoFPS - } - - return &H264Reader{ - reader: h, - file: f, - fps: fps, - }, nil -} - -func (r *H264Reader) NextFrame() ([]byte, uint32, error) { - nal, err := r.reader.NextNAL() - if err != nil { - if err == io.EOF { - return nil, 0, io.EOF - } - return nil, 0, err - } - - samples := uint32(videoClockRate / r.fps) - return nal.Data, samples, nil -} - -func (r *H264Reader) ClockRate() uint32 { - return videoClockRate -} - -func (r *H264Reader) Close() error { - return r.file.Close() -} diff --git a/pkg/pipeline/source/testfeeder/reader_ivf.go b/pkg/pipeline/source/testfeeder/reader_ivf.go deleted file mode 100644 index 6faed1a90..000000000 --- a/pkg/pipeline/source/testfeeder/reader_ivf.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import ( - "io" - "os" - - "github.com/pion/webrtc/v4/pkg/media/ivfreader" -) - -// IVFReader reads VP8 or VP9 frames from an IVF container file. -// The IVF container provides per-frame timestamps, so duration is computed from -// the difference between consecutive frame timestamps (converted to clock rate -// units via the IVF timebase). -type IVFReader struct { - reader *ivfreader.IVFReader - file *os.File - header *ivfreader.IVFFileHeader - lastPTS uint64 - firstRead bool -} - -// NewIVFReader opens an IVF file and returns a FrameReader. -func NewIVFReader(path string) (*IVFReader, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - reader, header, err := ivfreader.NewWith(f) - if err != nil { - f.Close() - return nil, err - } - - return &IVFReader{ - reader: reader, - file: f, - header: header, - firstRead: true, - }, nil -} - -func (r *IVFReader) NextFrame() ([]byte, uint32, error) { - payload, frameHeader, err := r.reader.ParseNextFrame() - if err != nil { - if err == io.EOF { - return nil, 0, io.EOF - } - return nil, 0, err - } - - var samples uint32 - if r.firstRead { - // First frame: use default frame duration based on the timebase. - // timebaseDenominator is fps (e.g., 24), timebaseNumerator is typically 1. - samples = videoClockRate * uint32(r.header.TimebaseNumerator) / uint32(r.header.TimebaseDenominator) - r.firstRead = false - } else { - // Duration = difference between this frame's timestamp and the last. - // IVF timestamps are in timebase units (numerator/denominator seconds). - diff := frameHeader.Timestamp - r.lastPTS - samples = uint32(diff * uint64(videoClockRate) * uint64(r.header.TimebaseNumerator) / uint64(r.header.TimebaseDenominator)) - if samples == 0 { - // Fallback for zero-duration frames. - samples = videoClockRate * uint32(r.header.TimebaseNumerator) / uint32(r.header.TimebaseDenominator) - } - } - r.lastPTS = frameHeader.Timestamp - - return payload, samples, nil -} - -func (r *IVFReader) ClockRate() uint32 { - return videoClockRate -} - -func (r *IVFReader) Close() error { - return r.file.Close() -} diff --git a/pkg/pipeline/source/testfeeder/reader_ogg.go b/pkg/pipeline/source/testfeeder/reader_ogg.go deleted file mode 100644 index a069469ba..000000000 --- a/pkg/pipeline/source/testfeeder/reader_ogg.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import ( - "encoding/binary" - "io" - "os" -) - -const ( - oggPageHeaderLen = 27 - oggHeaderSig = "OggS" - - // Opus uses 20ms frames at 48kHz = 960 samples per frame. - opusSamplesPerFrame = 960 -) - -// OGGReader reads individual Opus frames from an OGG container file. -// OGG pages can contain multiple Opus packets; this reader splits them -// using the OGG segment table so each NextFrame call returns exactly one -// Opus frame suitable for RTP packetization. -type OGGReader struct { - stream io.ReadCloser - - // Buffer of Opus packets extracted from the current OGG page. - // Each call to NextFrame pops one packet from this buffer. - packets [][]byte -} - -// NewOGGReader opens an OGG file containing Opus audio and returns a FrameReader. -func NewOGGReader(path string) (*OGGReader, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - - r := &OGGReader{stream: f} - - // Skip the OGG header pages (OpusHead + OpusTags). - // These are identified by the beginning-of-stream flag or by having - // granule position 0. We just skip until we find a page with audio data. - for { - packets, granule, err := r.readPage() - if err != nil { - f.Close() - return nil, err - } - // Audio pages have granule position > 0 - if granule > 0 { - r.packets = packets - break - } - } - - return r, nil -} - -func (r *OGGReader) NextFrame() ([]byte, uint32, error) { - for len(r.packets) == 0 { - packets, _, err := r.readPage() - if err != nil { - if err == io.EOF { - return nil, 0, io.EOF - } - return nil, 0, err - } - r.packets = packets - } - - pkt := r.packets[0] - r.packets = r.packets[1:] - return pkt, opusSamplesPerFrame, nil -} - -func (r *OGGReader) ClockRate() uint32 { - return opusClockRate -} - -func (r *OGGReader) Close() error { - return r.stream.Close() -} - -// readPage reads one OGG page and splits its payload into individual packets -// using the segment table. Returns the packets and the page's granule position. -func (r *OGGReader) readPage() ([][]byte, uint64, error) { - // Read page header (27 bytes) - header := make([]byte, oggPageHeaderLen) - if _, err := io.ReadFull(r.stream, header); err != nil { - return nil, 0, err - } - - if string(header[0:4]) != oggHeaderSig { - return nil, 0, io.ErrUnexpectedEOF - } - - granule := binary.LittleEndian.Uint64(header[6:14]) - segCount := int(header[26]) - - // Read segment table - segTable := make([]byte, segCount) - if _, err := io.ReadFull(r.stream, segTable); err != nil { - return nil, 0, err - } - - // Compute total payload size and read it - var totalSize int - for _, s := range segTable { - totalSize += int(s) - } - payload := make([]byte, totalSize) - if _, err := io.ReadFull(r.stream, payload); err != nil { - return nil, 0, err - } - - // Split payload into packets using the segment table. - // A packet spans consecutive segments; it ends when a segment is < 255. - var packets [][]byte - offset := 0 - packetStart := 0 - for _, s := range segTable { - offset += int(s) - if s < 255 { - // End of packet - if offset > packetStart { - packets = append(packets, payload[packetStart:offset]) - } - packetStart = offset - } - } - // Handle trailing packet that ends exactly on a 255-byte boundary - // (no terminating short segment). This is a continuation that spans - // to the next page — skip it for now. - - return packets, granule, nil -} diff --git a/pkg/pipeline/source/testfeeder/reader_slow.go b/pkg/pipeline/source/testfeeder/reader_slow.go deleted file mode 100644 index 2183629ee..000000000 --- a/pkg/pipeline/source/testfeeder/reader_slow.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import "time" - -// SlowReader wraps a FrameReader and adds a fixed delay before each frame read, -// simulating a source that produces data slower than the pipeline can consume. -// This is useful for testing that the non-live pipeline correctly handles -// asymmetric source rates via backpressure rather than dropping data. -type SlowReader struct { - inner FrameReader - delay time.Duration -} - -// NewSlowReader wraps an existing FrameReader with a per-frame delay. -func NewSlowReader(inner FrameReader, delay time.Duration) *SlowReader { - return &SlowReader{inner: inner, delay: delay} -} - -func (r *SlowReader) NextFrame() ([]byte, uint32, error) { - time.Sleep(r.delay) - return r.inner.NextFrame() -} - -func (r *SlowReader) ClockRate() uint32 { - return r.inner.ClockRate() -} - -func (r *SlowReader) Close() error { - return r.inner.Close() -} diff --git a/pkg/pipeline/source/testfeeder/reader_stalling.go b/pkg/pipeline/source/testfeeder/reader_stalling.go deleted file mode 100644 index cb6cc6c7a..000000000 --- a/pkg/pipeline/source/testfeeder/reader_stalling.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import "io" - -// StallingReader wraps a FrameReader and blocks forever after delivering a -// fixed number of frames. This simulates a track that has a gap in its -// recording — data stops arriving but no EOS or GAP event is sent. -type StallingReader struct { - inner FrameReader - maxFrames int - delivered int - stallCh chan struct{} // closed when the reader starts stalling - unblockCh chan struct{} // close this to unblock the stall (e.g. for cleanup) -} - -// NewStallingReader wraps an existing FrameReader. After maxFrames have been -// read, NextFrame blocks until unblockCh is closed (or forever if nil). -// stallCh is closed when the stall begins, allowing the test to detect it. -func NewStallingReader(inner FrameReader, maxFrames int) *StallingReader { - return &StallingReader{ - inner: inner, - maxFrames: maxFrames, - stallCh: make(chan struct{}), - unblockCh: make(chan struct{}), - } -} - -// Stalled returns a channel that is closed when the reader begins stalling. -func (r *StallingReader) Stalled() <-chan struct{} { - return r.stallCh -} - -// Unblock releases the stalled reader so the test can clean up. After -// unblocking, NextFrame returns io.EOF. -func (r *StallingReader) Unblock() { - select { - case <-r.unblockCh: - default: - close(r.unblockCh) - } -} - -func (r *StallingReader) NextFrame() ([]byte, uint32, error) { - if r.delivered >= r.maxFrames { - // Signal that we've started stalling. - select { - case <-r.stallCh: - default: - close(r.stallCh) - } - // Block until unblocked (test cleanup). - <-r.unblockCh - return nil, 0, io.EOF - } - - payload, samples, err := r.inner.NextFrame() - if err != nil { - return nil, 0, err - } - r.delivered++ - return payload, samples, nil -} - -func (r *StallingReader) ClockRate() uint32 { - return r.inner.ClockRate() -} - -func (r *StallingReader) Close() error { - r.Unblock() - return r.inner.Close() -} diff --git a/pkg/pipeline/source/testfeeder/source.go b/pkg/pipeline/source/testfeeder/source.go deleted file mode 100644 index bf96d062b..000000000 --- a/pkg/pipeline/source/testfeeder/source.go +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package testfeeder - -import ( - "fmt" - "sync" - "time" - - "github.com/frostbyte73/core" - "github.com/go-gst/go-gst/gst" - "github.com/go-gst/go-gst/gst/app" - "github.com/pion/webrtc/v4" - - "github.com/livekit/egress/pkg/config" - "github.com/livekit/egress/pkg/gstreamer" - "github.com/livekit/egress/pkg/types" - "github.com/livekit/protocol/logger" - lksdk "github.com/livekit/server-sdk-go/v2" -) - -// TestSource implements the source.Source interface by feeding pre-encoded media -// files through the GStreamer pipeline at maximum speed. It creates TrackSource -// objects, populates the PipelineConfig, and runs TrackFeeder goroutines after -// the pipeline starts. -// -// GStreamer must be initialized (gst.Init) before calling NewTestSource, since -// it creates appsrc elements. -// -// Usage: -// -// gst.Init(nil) -// ts, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ -// {Path: "audio.ogg", MimeType: types.MimeTypeOpus, PayloadType: 111}, -// {Path: "video.h264", MimeType: types.MimeTypeH264, PayloadType: 96, FPS: 30}, -// }) -// // conf.AudioTracks, conf.VideoTrack, etc. are now populated. -// // Build the pipeline, then call ts.Start() to begin feeding. -type TestSource struct { - conf *config.PipelineConfig - callbacks *gstreamer.Callbacks - - tracks []trackEntry - feeders []*TrackFeeder - startedAt int64 - - endRecording core.Fuse -} - -type trackEntry struct { - def TrackDef - ts *config.TrackSource - reader FrameReader -} - -// TrackDef describes a media file to feed into the pipeline. -type TrackDef struct { - // Path to the media file (H.264 annexb, VP8/VP9 IVF, or Opus OGG). - Path string - - // MimeType of the media (types.MimeTypeH264, MimeTypeVP8, MimeTypeOpus, etc.). - MimeType types.MimeType - - // PayloadType is the RTP payload type number. - PayloadType uint8 - - // FPS is the assumed frame rate for video files. Ignored for audio. - // Defaults to 30 if zero. - FPS int - - // TrackID is an optional identifier. Auto-generated if empty. - TrackID string - - // Reader optionally overrides the file-based reader. When set, Path is - // ignored and this reader is used directly. This allows wrapping readers - // (e.g. SlowReader) for testing. - Reader FrameReader -} - -// NewTestSource creates a TestSource that populates the PipelineConfig with -// TrackSource objects for each track definition. GStreamer must already be -// initialized. After this returns, the config is ready for BuildPipeline. -// Call Start() after the pipeline is built to begin feeding data. -func NewTestSource(conf *config.PipelineConfig, defs []TrackDef) (*TestSource, error) { - s := &TestSource{ - conf: conf, - } - - conf.Live = false - conf.SourceType = types.SourceTypeSDK - - for i, def := range defs { - if def.TrackID == "" { - def.TrackID = fmt.Sprintf("test_track_%d", i) - } - - reader := def.Reader - if reader == nil { - var err error - reader, err = openReader(def) - if err != nil { - s.closeReaders() - return nil, fmt.Errorf("opening %s: %w", def.Path, err) - } - } - - src, err := gst.NewElementWithName("appsrc", fmt.Sprintf("app_%s", def.TrackID)) - if err != nil { - s.closeReaders() - return nil, fmt.Errorf("creating appsrc: %w", err) - } - - ts := &config.TrackSource{ - TrackID: def.TrackID, - MimeType: def.MimeType, - PayloadType: webrtc.PayloadType(def.PayloadType), - ClockRate: reader.ClockRate(), - AppSrc: app.SrcFromElement(src), - } - - switch def.MimeType { - case types.MimeTypeOpus, types.MimeTypePCMU, types.MimeTypePCMA: - ts.TrackKind = lksdk.TrackKindAudio - conf.AudioEnabled = true - conf.AudioTranscoding = true - if conf.AudioOutCodec == "" { - if def.MimeType == types.MimeTypePCMU || def.MimeType == types.MimeTypePCMA { - conf.AudioOutCodec = types.MimeTypeOpus - } else { - conf.AudioOutCodec = def.MimeType - } - } - conf.AudioTracks = append(conf.AudioTracks, ts) - - case types.MimeTypeH264, types.MimeTypeVP8, types.MimeTypeVP9: - ts.TrackKind = lksdk.TrackKindVideo - conf.VideoEnabled = true - conf.VideoInCodec = def.MimeType - if conf.VideoOutCodec == "" { - conf.VideoOutCodec = def.MimeType - } - if conf.VideoInCodec != conf.VideoOutCodec { - conf.VideoDecoding = true - if len(conf.GetEncodedOutputs()) > 0 { - conf.VideoEncoding = true - } - } - conf.VideoTrack = ts - - default: - s.closeReaders() - return nil, fmt.Errorf("unsupported mime type: %s", def.MimeType) - } - - s.tracks = append(s.tracks, trackEntry{ - def: def, - ts: ts, - reader: reader, - }) - } - - return s, nil -} - -// --- source.Source interface --- - -// StartRecording returns a channel that closes immediately, signaling to the -// controller that the source is ready. It also starts feeding all tracks -// concurrently in background goroutines. EndRecording fires when all feeders -// have finished and sent EOS. -func (s *TestSource) StartRecording() <-chan struct{} { - s.startedAt = time.Now().UnixNano() - - // Launch feeders in a goroutine — they will wait for the pipeline to - // reach PLAYING state before pushing any data, so StartRecording can - // return immediately to unblock the controller's Run loop. - go s.feedAll() - - ch := make(chan struct{}) - close(ch) - return ch -} - -func (s *TestSource) feedAll() { - // Wait for the pipeline to reach PAUSED before pushing data. - // The audiotestsrc (is-live=true) prerolls the pipeline. - <-s.callbacks.PipelinePaused() - - var wg sync.WaitGroup - for i := range s.tracks { - te := &s.tracks[i] - - feeder, err := NewTrackFeeder(TrackFeederParams{ - AppSrc: te.ts.AppSrc, - MimeType: te.def.MimeType, - PayloadType: te.def.PayloadType, - SSRC: uint32(1000 + i), - Reader: te.reader, - }) - if err != nil { - logger.Errorw("failed to create feeder", err, "trackID", te.def.TrackID) - continue - } - s.feeders = append(s.feeders, feeder) - - wg.Add(1) - go func() { - defer wg.Done() - if err := feeder.Feed(); err != nil { - logger.Errorw("feeder error", err, "trackID", te.def.TrackID) - } - }() - } - - wg.Wait() - logger.Infow("all test feeders finished, EOS sent on all appsrc elements") - s.endRecording.Break() -} - -func (s *TestSource) EndRecording() <-chan struct{} { - return s.endRecording.Watch() -} - -func (s *TestSource) GetStartedAt() int64 { - return s.startedAt -} - -func (s *TestSource) GetEndedAt() int64 { - return time.Now().UnixNano() -} - -func (s *TestSource) Close() { - s.closeReaders() -} - -// --- source.TimeAware interface --- - -func (s *TestSource) SetTimeProvider(_ gstreamer.TimeProvider) { - // No-op. Non-live pipeline doesn't need time feedback. -} - -// SetCallbacks sets the pipeline callbacks. Must be called before the pipeline -// starts (before Run). The controller exposes callbacks via Callbacks(). -func (s *TestSource) SetCallbacks(cb *gstreamer.Callbacks) { - s.callbacks = cb -} - -// --- helpers --- - -func (s *TestSource) closeReaders() { - for _, te := range s.tracks { - if te.reader != nil { - te.reader.Close() - } - } -} - -func openReader(def TrackDef) (FrameReader, error) { - switch def.MimeType { - case types.MimeTypeH264: - return NewH264Reader(def.Path, def.FPS) - case types.MimeTypeVP8, types.MimeTypeVP9: - return NewIVFReader(def.Path) - case types.MimeTypeOpus: - return NewOGGReader(def.Path) - default: - return nil, fmt.Errorf("no reader for %s", def.MimeType) - } -} diff --git a/pkg/pipeline/source/testfeeder/source_test.go b/pkg/pipeline/source/testfeeder/source_test.go deleted file mode 100644 index 324f10f83..000000000 --- a/pkg/pipeline/source/testfeeder/source_test.go +++ /dev/null @@ -1,583 +0,0 @@ -// Copyright 2024 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build integration - -package testfeeder_test - -import ( - "context" - "os" - "path/filepath" - "testing" - "time" - - "github.com/go-gst/go-gst/gst" - "github.com/stretchr/testify/require" - - "github.com/livekit/egress/pkg/config" - "github.com/livekit/egress/pkg/pipeline" - "github.com/livekit/egress/pkg/pipeline/source/testfeeder" - "github.com/livekit/egress/pkg/types" - "github.com/livekit/protocol/livekit" - "github.com/livekit/protocol/logger" - "github.com/livekit/protocol/observability/storageobs" -) - -func init() { - os.Setenv("GST_DEBUG", "audiomixer:6,aggregator:6,queue:5,audiotestsrc:5") - logger.InitFromConfig(&logger.Config{Level: "debug"}, "testfeeder") -} - -// mediaSamplesDir returns the path to the media-samples directory. -// Set MEDIA_SAMPLES_DIR env var to override, otherwise defaults to the -// media-samples directory at the repo root. -func mediaSamplesDir(t *testing.T) string { - if dir := os.Getenv("MEDIA_SAMPLES_DIR"); dir != "" { - return dir - } - dir := repoRoot(t) - samples := filepath.Join(dir, "media-samples") - if _, err := os.Stat(samples); err != nil { - t.Skipf("media-samples not found at %s (set MEDIA_SAMPLES_DIR)", samples) - } - return samples -} - -// repoRoot walks up from the current file's directory until it finds go.mod. -func repoRoot(t *testing.T) string { - t.Helper() - dir, err := os.Getwd() - require.NoError(t, err) - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - return dir - } - parent := filepath.Dir(dir) - if parent == dir { - t.Fatal("could not find repo root (no go.mod found)") - } - dir = parent - } -} - -// newTestPipelineConfig creates a minimal PipelineConfig suitable for pipeline -// construction. Track info is populated by TestSource; this sets up everything -// else the pipeline needs (latency, output config, etc.). -func newTestPipelineConfig(t *testing.T, outputPath string, outputType types.OutputType) *config.PipelineConfig { - t.Helper() - - tmpDir := t.TempDir() - - p := &config.PipelineConfig{ - BaseConfig: config.BaseConfig{ - Logging: &logger.Config{Level: "debug"}, - Latency: config.LatencyConfig{ - PipelineLatency: 3 * time.Second, - }, - }, - TmpDir: tmpDir, - Outputs: make(map[types.EgressType][]config.OutputConfig), - StorageReporter: storageobs.NewNoopProjectReporter(), - } - - p.AudioConfig = config.AudioConfig{ - AudioBitrate: 128, - AudioFrequency: 48000, - } - - p.Info = &livekit.EgressInfo{ - EgressId: "test-feeder-integration", - Status: livekit.EgressStatus_EGRESS_STARTING, - StartedAt: time.Now().UnixNano(), - UpdatedAt: time.Now().UnixNano(), - } - - p.Outputs[types.EgressTypeFile] = []config.OutputConfig{ - &config.FileConfig{ - FileInfo: &livekit.FileInfo{}, - LocalFilepath: outputPath, - StorageFilepath: outputPath, - DisableManifest: true, - }, - } - p.GetFileConfig().OutputType = outputType - - return p -} - -// TestAudioOnlyOGG tests the full non-live pipeline with a single Opus audio -// track producing an OGG file. This is the simplest case: one appsrc, one -// depayloader, one decoder, one encoder, one muxer, one file sink. -func TestAudioOnlyOGG(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-output.ogg" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) - - // Initialize GStreamer before creating source (appsrc needs it) - gst.Init(nil) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - Path: filepath.Join(samples, "SolLevante.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - }) - require.NoError(t, err) - - // Verify config was populated - require.True(t, conf.AudioEnabled) - require.False(t, conf.Live) - require.Len(t, conf.AudioTracks, 1) - - // Create pipeline via controller - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - // Wire callbacks so feeders know when pipeline is ready - src.SetCallbacks(ctrl.Callbacks()) - - // Run pipeline to completion — this blocks until EOS. - // The controller calls src.StartRecording() which starts the feeders. - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - // Verify output file exists and has content - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestAsymmetricAudioRates tests that the non-live pipeline correctly handles -// two audio sources producing at different rates. One track pushes at full speed -// while the other is throttled to roughly realtime (20ms sleep per 20ms Opus -// frame). The audiomixer should wait for the slow source via backpressure — no -// data loss, valid output. -func TestAsymmetricAudioRates(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-asymmetric.ogg" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) - conf.Latency.AudioMixerLatency = 2750 * time.Millisecond - - gst.Init(nil) - - // Open the slow reader manually so we can wrap it. - // 20ms delay per frame ≈ realtime playback speed, roughly 2x slower than - // the fast track which pushes as fast as the pipeline accepts. - // Using a different audio file so both tracks are audibly distinguishable - // in the output. - slowInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) - require.NoError(t, err) - slowReader := testfeeder.NewSlowReader(slowInner, 20*time.Millisecond) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - TrackID: "fast_track", - Path: filepath.Join(samples, "SolLevante.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - { - TrackID: "slow_track", - MimeType: types.MimeTypeOpus, - PayloadType: 111, - Reader: slowReader, - }, - }) - require.NoError(t, err) - - require.Len(t, conf.AudioTracks, 2) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - t.Logf("slow track (120s audio at realtime) gates the pipeline, got %v wall clock", elapsed) - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestMixerStallsOnGap proves that the non-live audiomixer blocks forever when -// one track stops producing data without sending EOS or a GAP event. This -// demonstrates why non-live sources must detect gaps in their input data and -// inject GAP events. -func TestMixerStallsOnGap(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-stall.ogg" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) - conf.Latency.AudioMixerLatency = 2750 * time.Millisecond - - gst.Init(nil) - - // The stalling reader delivers 50 frames (~1s of Opus) then blocks forever, - // simulating a track with a gap in its recording. - stallingInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) - require.NoError(t, err) - stallingReader := testfeeder.NewStallingReader(stallingInner, 50) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - TrackID: "normal_track", - Path: filepath.Join(samples, "SolLevante.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - { - TrackID: "stalling_track", - MimeType: types.MimeTypeOpus, - PayloadType: 111, - Reader: stallingReader, - }, - }) - require.NoError(t, err) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - // Run the pipeline in a goroutine since it will stall. - done := make(chan *livekit.EgressInfo, 1) - go func() { - done <- ctrl.Run(ctx) - }() - - // Wait for the stalling reader to confirm it has stopped producing data. - select { - case <-stallingReader.Stalled(): - t.Log("stalling reader has stopped producing data after 50 frames") - case <-time.After(30 * time.Second): - t.Fatal("timed out waiting for stalling reader to stall") - } - - // The pipeline should NOT complete within 5 seconds — the mixer is blocked - // waiting for the stalling track. - select { - case info := <-done: - t.Fatalf("pipeline should have stalled but completed with status: %s", info.Status) - case <-time.After(5 * time.Second): - t.Log("CONFIRMED: pipeline is stalled — mixer is blocked waiting for missing data") - } - - // Clean up: unblock the stalling reader so the pipeline can shut down. - stallingReader.Unblock() - - select { - case info := <-done: - t.Logf("pipeline finished after unblock, status: %s", info.Status) - case <-time.After(30 * time.Second): - t.Fatal("pipeline did not finish after unblocking stalling reader") - } -} - -// TestGapEventUnblocksMixer proves that sending a GAP event on a track allows -// the audiomixer to proceed with silence instead of blocking forever. This is -// the fix for the stall demonstrated in TestMixerStallsOnGap. -// -// Track 1: full SolLevante.ogg (~28s), pushes at full speed. -// Track 2: 50 frames (~1s) of audio, then a 10s GAP event, then continues. -// -// The pipeline should complete quickly (faster than realtime) without stalling. -func TestGapEventUnblocksMixer(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-gap-event.ogg" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) - conf.Latency.AudioMixerLatency = 2750 * time.Millisecond - - gst.Init(nil) - - // After 50 frames (~1s), inject a 10s GAP event, then continue with - // remaining frames. The mixer should fill silence for track 2 during - // the gap and keep mixing track 1. - gappingInner, err := testfeeder.NewOGGReader(filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg")) - require.NoError(t, err) - gappingReader := testfeeder.NewGappingReader(gappingInner, 50, 10*time.Second) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - TrackID: "normal_track", - Path: filepath.Join(samples, "SolLevante.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - { - TrackID: "gapping_track", - MimeType: types.MimeTypeOpus, - PayloadType: 111, - Reader: gappingReader, - }, - }) - require.NoError(t, err) - - require.Len(t, conf.AudioTracks, 2) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - // The pipeline should complete much faster than realtime — if it took - // >15s, something is wrong (the gap event didn't unblock the mixer). - require.Less(t, elapsed, 15*time.Second, - "pipeline should complete faster than realtime; GAP event may not have worked") - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestRoomCompositeAudioOnly tests the non-live pipeline with two Opus audio -// tracks mixed through audiomixer, simulating an audio-only room composite with -// two participants. Both tracks are decoded, mixed, and re-encoded to OGG/Opus. -func TestRoomCompositeAudioOnly(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-room-audio.ogg" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeOGG) - conf.Latency.AudioMixerLatency = 2750 * time.Millisecond - - gst.Init(nil) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - TrackID: "participant_1_audio", - Path: filepath.Join(samples, "SolLevante.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - { - TrackID: "participant_2_audio", - Path: filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - }) - require.NoError(t, err) - - require.True(t, conf.AudioEnabled) - require.False(t, conf.VideoEnabled) - require.False(t, conf.Live) - require.Len(t, conf.AudioTracks, 2) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestAudioVideoMP4 tests the non-live pipeline with both an Opus audio track -// and an H.264 video track producing an MP4 file. This simulates a participant -// egress: audio goes through Opus→decode→AAC encode, video passes through as -// H.264. -func TestAudioVideoMP4(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-av.mp4" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeMP4) - conf.VideoConfig = config.VideoConfig{ - VideoProfile: types.ProfileMain, - Width: 1920, - Height: 1080, - Depth: 24, - Framerate: 25, - VideoBitrate: 3000, - } - conf.AudioOutCodec = types.MimeTypeAAC - conf.AudioTranscoding = true - - gst.Init(nil) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - Path: filepath.Join(samples, "avsync_minmotion_livekit_audio_48k_120s.ogg"), - MimeType: types.MimeTypeOpus, - PayloadType: 111, - }, - { - Path: filepath.Join(samples, "avsync_minmotion_livekit_video_1080p25_120s.h264"), - MimeType: types.MimeTypeH264, - PayloadType: 96, - FPS: 25, - }, - }) - require.NoError(t, err) - - require.True(t, conf.AudioEnabled) - require.True(t, conf.VideoEnabled) - require.False(t, conf.Live) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestVideoOnlyH264MP4 tests the non-live pipeline with a single H.264 video -// track producing an MP4 file. No audio — just video appsrc → depay → mux → file. -func TestVideoOnlyH264MP4(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-video-h264.mp4" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeMP4) - conf.VideoConfig = config.VideoConfig{ - VideoProfile: types.ProfileMain, - Width: 1920, - Height: 1080, - Depth: 24, - Framerate: 25, - VideoBitrate: 3000, - } - - gst.Init(nil) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - Path: filepath.Join(samples, "avsync_minmotion_livekit_video_1080p25_120s.h264"), - MimeType: types.MimeTypeH264, - PayloadType: 96, - FPS: 25, - }, - }) - require.NoError(t, err) - - require.True(t, conf.VideoEnabled) - require.False(t, conf.Live) - require.NotNil(t, conf.VideoTrack) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} - -// TestVideoOnlyVP8WebM tests the non-live pipeline with a single VP8 video -// track producing a WebM file. -func TestVideoOnlyVP8WebM(t *testing.T) { - samples := mediaSamplesDir(t) - outputPath := "/tmp/testfeeder-video-vp8.webm" - - conf := newTestPipelineConfig(t, outputPath, types.OutputTypeWebM) - conf.VideoConfig = config.VideoConfig{ - VideoProfile: types.ProfileMain, - Width: 1920, - Height: 1080, - Depth: 24, - Framerate: 24, - VideoBitrate: 3000, - } - - gst.Init(nil) - - src, err := testfeeder.NewTestSource(conf, []testfeeder.TrackDef{ - { - Path: filepath.Join(samples, "SolLevante-vp8.ivf"), - MimeType: types.MimeTypeVP8, - PayloadType: 96, - }, - }) - require.NoError(t, err) - - require.True(t, conf.VideoEnabled) - require.False(t, conf.Live) - require.NotNil(t, conf.VideoTrack) - - ctx := context.Background() - ctrl, err := pipeline.NewWithSource(ctx, conf, src) - require.NoError(t, err) - - src.SetCallbacks(ctrl.Callbacks()) - - start := time.Now() - info := ctrl.Run(ctx) - elapsed := time.Since(start) - - t.Logf("pipeline completed in %v, status: %s", elapsed, info.Status) - - stat, err := os.Stat(outputPath) - require.NoError(t, err, "output file should exist") - require.Greater(t, stat.Size(), int64(0), "output file should not be empty") - - t.Logf("output file: %s (%d bytes)", outputPath, stat.Size()) -} From 793534a0b58c9996a8195bd4406e60e2e953d658 Mon Sep 17 00:00:00 2001 From: Milos Pesic Date: Sun, 19 Apr 2026 11:10:17 +0200 Subject: [PATCH 9/9] get rid of code duplication by passing source builder --- pkg/pipeline/controller.go | 52 +++++++++++++++----------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index bae2656ce..d82471a32 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -96,6 +96,11 @@ type controllerStats struct { droppedVideoBuffersByQueue map[string]uint64 } +// SourceBuilder constructs a pipeline source. It receives the controller's +// callbacks so the source can synchronize on GstReady; custom sources that +// don't need gst synchronization can ignore the argument. +type SourceBuilder func(callbacks *gstreamer.Callbacks) (source.Source, error) + var ( tracer = otel.Tracer("github.com/livekit/egress/pkg/pipeline") ) @@ -104,6 +109,21 @@ func New(ctx context.Context, conf *config.PipelineConfig, ipcServiceClient ipc. ctx, span := tracer.Start(ctx, "Pipeline.New") defer span.End() + return NewWithSource(ctx, conf, ipcServiceClient, func(callbacks *gstreamer.Callbacks) (source.Source, error) { + return source.New(ctx, conf, callbacks) + }) +} + +// NewWithSource creates a Controller using the given SourceBuilder. The builder +// runs after the controller has been constructed and receives the controller's +// Callbacks, so the source can share GstReady with the pipeline. Use this when +// the source isn't the standard source.New (testfeeder, replay export, etc.). +func NewWithSource( + ctx context.Context, + conf *config.PipelineConfig, + ipcServiceClient ipc.EgressServiceClient, + srcBuilder SourceBuilder, +) (*Controller, error) { c := newController(conf, ipcServiceClient) // initialize gst @@ -115,42 +135,12 @@ func New(ctx context.Context, conf *config.PipelineConfig, ipcServiceClient ipc. close(c.callbacks.GstReady) }() - // create source - var err error - c.src, err = source.New(ctx, conf, c.callbacks) + src, err := srcBuilder(c.callbacks) if err != nil { return nil, err } - - // create pipeline - <-c.callbacks.GstReady - if err = c.BuildPipeline(); err != nil { - c.src.Close() - return nil, err - } - - return c, nil -} - -// NewWithSource creates a Controller with a pre-built source. This is used when -// the source is constructed externally (e.g. TestSource for non-live pipeline -// testing, or ReplaySource for offline export). The source must have already -// populated the PipelineConfig with track information before calling this. -func NewWithSource(ctx context.Context, conf *config.PipelineConfig, src source.Source) (*Controller, error) { - ctx, span := tracer.Start(ctx, "Pipeline.NewWithSource") - defer span.End() - c := newController(conf, nil) c.src = src - // initialize gst - go func() { - _, span := tracer.Start(ctx, "gst.Init") - defer span.End() - gst.Init(nil) - gst.SetLogFunction(c.gstLog) - close(c.callbacks.GstReady) - }() - // create pipeline <-c.callbacks.GstReady if err := c.BuildPipeline(); err != nil {