diff --git a/ffmpeg/ffmpeg.go b/ffmpeg/ffmpeg.go index 7739067615..bb5f28f6a2 100755 --- a/ffmpeg/ffmpeg.go +++ b/ffmpeg/ffmpeg.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "os" "path" "path/filepath" @@ -118,6 +119,8 @@ type MediaInfo struct { Frames int Pixels int64 DetectData DetectData + Width int + Height int } type TranscodeResults struct { @@ -300,6 +303,42 @@ func GetCodecInfoBytes(data []byte) (CodecStatus, MediaFormatInfo, error) { return status, format, err } +func GetDecoderStatsBytes(data []byte) (*MediaInfo, error) { + // write the data to a temp file + tempfile, err := ioutil.TempFile("", "") + if err != nil { + return nil, fmt.Errorf("error creating temp file for pixels verification: %w", err) + } + defer os.Remove(tempfile.Name()) + + if _, err := tempfile.Write(data); err != nil { + tempfile.Close() + return nil, fmt.Errorf("error writing temp file for pixels verification: %w", err) + } + + if err = tempfile.Close(); err != nil { + return nil, fmt.Errorf("error closing temp file for pixels verification: %w", err) + } + + mi, err := GetDecoderStats(tempfile.Name()) + if err != nil { + return nil, err + } + + return mi, nil +} + +// Calculates media file stats by fully decoding it. Use GetCodecInfo, if you need +// metadata from the start of the container. +func GetDecoderStats(fname string) (*MediaInfo, error) { + in := &TranscodeOptionsIn{Fname: fname} + res, err := Transcode3(in, nil) + if err != nil { + return nil, err + } + return &res.Decoded, nil +} + // HasZeroVideoFrameBytes opens video and returns true if it has video stream with 0-frame func HasZeroVideoFrameBytes(data []byte) (bool, error) { if len(data) == 0 { @@ -1016,6 +1055,8 @@ func (t *Transcoder) Transcode(input *TranscodeOptionsIn, ps []TranscodeOptions) dec := MediaInfo{ Frames: int(decoded.frames), Pixels: int64(decoded.pixels), + Width: int(decoded.width), + Height: int(decoded.height), } return &TranscodeResults{Encoded: tr, Decoded: dec}, nil } diff --git a/ffmpeg/ffmpeg_test.go b/ffmpeg/ffmpeg_test.go index 08ed5c17d2..83249c7047 100644 --- a/ffmpeg/ffmpeg_test.go +++ b/ffmpeg/ffmpeg_test.go @@ -516,6 +516,34 @@ nb_read_frames=%d } } +func TestFuzzyMatchMediaInfo(t *testing.T) { + actualInfo := MediaInfo{Frames: 60, Pixels: 20736000, Width: 720, Height: 480} + // all match + result := FuzzyMatchMediaInfo(actualInfo, 20736000) + require.True(t, result) + // custom profile, pixel count mismatch reported transcoded < actual within tolerance - pass + result = FuzzyMatchMediaInfo(actualInfo, 717*480*60) + require.True(t, result) + // custom profile, reported transcoded > actual - fail + result = FuzzyMatchMediaInfo(actualInfo, 20736001) + require.False(t, result) + // custom profile, too significant difference - fail + result = FuzzyMatchMediaInfo(actualInfo, 716*480*60) + require.False(t, result) +} + +func TestGetDecoderStats(t *testing.T) { + wd, _ := os.Getwd() + stats, err := GetDecoderStats(path.Join(wd, "../transcoder/test.ts")) + require.NoError(t, err) + require.Equal(t, 1280, stats.Width) + require.Equal(t, 720, stats.Height) + require.Equal(t, 480, stats.Frames) + // check for correct error + _, err = GetDecoderStats(path.Join(wd, "foo")) + require.EqualError(t, err, "TranscoderInvalidVideo") +} + func TestTranscoder_StatisticsAspectRatio(t *testing.T) { // Check that we correctly account for aspect ratio adjustments // Eg, the transcoded resolution we receive may be smaller than diff --git a/ffmpeg/nvidia_test.go b/ffmpeg/nvidia_test.go index de70a2f510..f2f877f8fd 100755 --- a/ffmpeg/nvidia_test.go +++ b/ffmpeg/nvidia_test.go @@ -732,13 +732,13 @@ func TestNvidia_DetectionFreq(t *testing.T) { detectionFreq(t, Nvidia, "0") } -func portraitTest(t *testing.T, input string, checkResults bool, profiles []VideoProfile) error { +func resolutionsAndPixelsTest(t *testing.T, input string, checkResults bool, profiles []VideoProfile) error { wd, err := os.Getwd() require.NoError(t, err) outName := func(index int, resolution string) string { - return path.Join(wd, "..", "data", fmt.Sprintf("%s_%d_%s.ts", strings.ReplaceAll(input, ".", "_"), index, resolution)) + return path.Join(wd, "..", "data", fmt.Sprintf("%s_%d_%s.ts", strings.ReplaceAll(path.Base(input), ".", "_"), index, resolution)) } - fname := path.Join(wd, "..", "data", input) + fname := path.Join(wd, input) in := &TranscodeOptionsIn{Fname: fname, Accel: Nvidia} out := make([]TranscodeOptions, 0, len(profiles)) outFilenames := make([]string, 0, len(profiles)) @@ -752,37 +752,64 @@ func portraitTest(t *testing.T, input string, checkResults bool, profiles []Vide }) outFilenames = append(outFilenames, filename) } - _, resultErr := Transcode3(in, out) + nvidiaTranscodeRes, resultErr := Transcode3(in, out) if resultErr == nil && checkResults { - for _, filename := range outFilenames { + for i, filename := range outFilenames { outInfo, err := os.Stat(filename) if os.IsNotExist(err) { require.NoError(t, err, fmt.Sprintf("output missing %s", filename)) } else { defer os.Remove(filename) + // check size + require.NotEqual(t, outInfo.Size(), 0, "must produce output %s", filename) + // software decode to get pixel counts for validation + cpuDecodeRes, cpuErr := Transcode3(&TranscodeOptionsIn{Fname: filename}, nil) + require.NoError(t, cpuErr, "Software decoder error") + fuzzyMatchResult := FuzzyMatchMediaInfo(cpuDecodeRes.Decoded, nvidiaTranscodeRes.Encoded[i].Pixels) + require.True(t, fuzzyMatchResult, "GPU encoder and CPU decoder pixel count mismatch for profile %s: %d vs %d", + profiles[i].Name, cpuDecodeRes.Decoded.Pixels, nvidiaTranscodeRes.Encoded[i].Pixels) } - require.NotEqual(t, outInfo.Size(), 0, "must produce output %s", filename) } } return resultErr } -func TestTranscoder_Portrait(t *testing.T) { - hevc := VideoProfile{Name: "P240p30fps16x9", Bitrate: "600k", Framerate: 30, AspectRatio: "16:9", Resolution: "426x240", Encoder: H265} +func TestTranscoder_ResolutionsAndPixels(t *testing.T) { + hevcPortrait := VideoProfile{Name: "P240p30fps16x9", Bitrate: "600k", Framerate: 30, AspectRatio: "16:9", Resolution: "426x240", Encoder: H265} - // Usuall portrait input sample - require.NoError(t, portraitTest(t, "portrait.ts", true, []VideoProfile{ - P360p30fps16x9, hevc, P144p30fps16x9, + commonProfiles := []VideoProfile{ + P144p30fps16x9, P240p30fps16x9, P360p30fps16x9, P720p60fps16x9, + P240p30fps4x3, P360p30fps4x3, P720p30fps4x3, + } + + commonProfilesHevc := func(ps []VideoProfile) []VideoProfile { + var res []VideoProfile + for _, p := range ps { + p.Encoder = H265 + res = append(res, p) + } + return res + }(commonProfiles) + + // Standard input sample to standard resolutions + require.NoError(t, resolutionsAndPixelsTest(t, "../transcoder/test_short.ts", true, commonProfiles)) + + // Standard input sample to standard resolutions HEVC + require.NoError(t, resolutionsAndPixelsTest(t, "../transcoder/test_short.ts", true, commonProfilesHevc)) + + // Usual portrait input sample + require.NoError(t, resolutionsAndPixelsTest(t, "../data/portrait.ts", true, []VideoProfile{ + P360p30fps16x9, hevcPortrait, P144p30fps16x9, })) // Reported as not working sample, but transcoding works as expected - require.NoError(t, portraitTest(t, "videotest.mp4", true, []VideoProfile{ - P360p30fps16x9, hevc, P144p30fps16x9, + require.NoError(t, resolutionsAndPixelsTest(t, "../data/videotest.mp4", true, []VideoProfile{ + P360p30fps16x9, hevcPortrait, P144p30fps16x9, })) // Created one sample that is impossible to resize and fit within encoder limits and still keep aspect ratio: notPossible := VideoProfile{Name: "P8K1x250", Bitrate: "6000k", Framerate: 30, AspectRatio: "1:250", Resolution: "250x62500", Encoder: H264} - err := portraitTest(t, "vertical-sample.ts", true, []VideoProfile{notPossible}) + err := resolutionsAndPixelsTest(t, "vertical-sample.ts", true, []VideoProfile{notPossible}) // We expect error require.Error(t, err) // Error should be `profile 250x62500 size out of bounds 146x146-4096x4096 input=16x4000 adjusted 250x62500 or 16x4096` diff --git a/ffmpeg/sign_nvidia_test.go b/ffmpeg/sign_nvidia_test.go index 7cb985a32c..2d7d07bc99 100644 --- a/ffmpeg/sign_nvidia_test.go +++ b/ffmpeg/sign_nvidia_test.go @@ -13,8 +13,8 @@ import ( "testing" ) -const SignCompareMaxFalseNegativeRate = 0.01; -const SignCompareMaxFalsePositiveRate = 0.15; +const SignCompareMaxFalseNegativeRate = 0.01 +const SignCompareMaxFalsePositiveRate = 0.15 func TestNvidia_SignDataCreate(t *testing.T) { _, dir := setupTest(t) diff --git a/ffmpeg/transcoder.c b/ffmpeg/transcoder.c index b9ba5672c0..0da8413b08 100755 --- a/ffmpeg/transcoder.c +++ b/ffmpeg/transcoder.c @@ -654,6 +654,11 @@ int transcode2(struct transcode_thread *h, if (ret < 0) LPMS_ERR_BREAK("Flushing failed"); ist = ictx->ic->streams[stream_index]; if (AVMEDIA_TYPE_VIDEO == ist->codecpar->codec_type) { + // assume resolution won't change mid-segment + if (!decoded_results->frames) { + decoded_results->width = iframe->width; + decoded_results->height = iframe->height; + } handle_video_frame(h, ist, decoded_results, iframe); } else if (AVMEDIA_TYPE_AUDIO == ist->codecpar->codec_type) { handle_audio_frame(h, ist, decoded_results, iframe); @@ -743,6 +748,11 @@ int transcode(struct transcode_thread *h, // width / height will be zero for pure streamcopy (no decoding) decoded_results->frames += dframe->width && dframe->height; decoded_results->pixels += dframe->width * dframe->height; + // assume resolution won't change mid-segment + if (decoded_results->frames == 1) { + decoded_results->width = dframe->width; + decoded_results->height = dframe->height; + } has_frame = has_frame && dframe->width && dframe->height; if (has_frame) last_frame = ictx->last_frame_v; } else if (AVMEDIA_TYPE_AUDIO == ist->codecpar->codec_type) { diff --git a/ffmpeg/transcoder.h b/ffmpeg/transcoder.h index 8e3177dc85..6e7b9ef781 100755 --- a/ffmpeg/transcoder.h +++ b/ffmpeg/transcoder.h @@ -75,6 +75,8 @@ typedef struct { int64_t pixels; //for scene classification float probs[MAX_CLASSIFY_SIZE];//probability + int width; + int height; } output_results; enum LPMSLogLevel { diff --git a/ffmpeg/videoprofile.go b/ffmpeg/videoprofile.go index a606b5f59b..8ff92befab 100644 --- a/ffmpeg/videoprofile.go +++ b/ffmpeg/videoprofile.go @@ -3,6 +3,7 @@ package ffmpeg import ( "encoding/json" "fmt" + "math" "strconv" "strings" "time" @@ -290,3 +291,14 @@ func ParseProfiles(injson []byte) ([]VideoProfile, error) { } return ParseProfilesFromJsonProfileArray(decodedJson.Profiles) } + +// checks whether the video's MediaInfo is plausible, given reported pixel count +func FuzzyMatchMediaInfo(actualInfo MediaInfo, transcodedPixelCount int64) bool { + // apply tolerance to larger dimension to account for portrait resolutions, and calculate max pixel mismatch with smaller dimension + smallerDim := int(math.Min(float64(actualInfo.Width), float64(actualInfo.Height))) + tol := 3 + pixelDiffTol := int64(tol * smallerDim * actualInfo.Frames) + pixelDiff := actualInfo.Pixels - transcodedPixelCount + // it should never report *more* pixels encoded, than rendition actually has + return pixelDiff >= 0 && pixelDiff <= pixelDiffTol +} diff --git a/transcoder/test_short.ts b/transcoder/test_short.ts new file mode 100644 index 0000000000..33b0c7ca5d Binary files /dev/null and b/transcoder/test_short.ts differ