From 9c12800d420b0e04cae073a73e2f1fcadc09fd31 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sun, 30 Jun 2024 02:23:06 +0300 Subject: [PATCH 01/20] avfilter/vf_libvmaf: Propagatin metadata --- libavfilter/vf_libvmaf.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 2c99d792afd06..4148b8d5e70f6 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -56,6 +56,7 @@ typedef struct LIBVMAFContext { int n_subsample; char *model_cfg; char *feature_cfg; + AVDictionary *metadata; VmafContext *vmaf; VmafModel **model; unsigned model_cnt; @@ -105,6 +106,28 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } +typedef struct MetaStruct { + AVDictionary **metadata; + char comp; +} MetaStruct; + +//static void set_meta(AVDictionary **metadata, const char *key, char comp, float d) +static void set_meta(void *data, const char *key, double d) +{ + if (!data) return; + MetaStruct *meta = data; + av_log(NULL, AV_LOG_ERROR, "key: %s, value: %f\n", key, d); + char value[128]; + snprintf(value, sizeof(value), "%f", d); + if (meta->comp) { + char key2[128]; + snprintf(key2, sizeof(key2), "%s%c", key, meta->comp); + av_dict_set(meta->metadata, key2, value, 0); + } else { + av_dict_set(meta->metadata, key, value, 0); + } +} + static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc) { const int bytes_per_value = bpc > 8 ? 2 : 1; @@ -431,10 +454,18 @@ static av_cold int init(AVFilterContext *ctx) LIBVMAFContext *s = ctx->priv; int err = 0; + MetaStruct meta_data = { + .metadata = &s->metadata, + .comp = 0, + }; + VmafConfiguration cfg = { .log_level = log_level_map(av_log_get_level()), .n_subsample = s->n_subsample, .n_threads = s->n_threads, + .callback = set_meta, + .meta_data = &meta_data, + .meta_data_sz = sizeof(meta_data), }; err = vmaf_init(&s->vmaf, cfg); From d938b529b6e4a992792162d234e77cfb62979dee Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sat, 6 Jul 2024 16:07:57 +0300 Subject: [PATCH 02/20] changes according to new api in libvmaf --- libavfilter/vf_libvmaf.c | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 4148b8d5e70f6..790b88cfbf41e 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -46,6 +46,11 @@ #include "libavutil/hwcontext_cuda_internal.h" #endif +typedef struct MetaStruct { + AVDictionary **metadata; + char comp; +} MetaStruct; + typedef struct LIBVMAFContext { const AVClass *class; FFFrameSync fs; @@ -57,6 +62,7 @@ typedef struct LIBVMAFContext { char *model_cfg; char *feature_cfg; AVDictionary *metadata; + MetaStruct *meta_data; VmafContext *vmaf; VmafModel **model; unsigned model_cnt; @@ -106,11 +112,6 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } -typedef struct MetaStruct { - AVDictionary **metadata; - char comp; -} MetaStruct; - //static void set_meta(AVDictionary **metadata, const char *key, char comp, float d) static void set_meta(void *data, const char *key, double d) { @@ -454,24 +455,34 @@ static av_cold int init(AVFilterContext *ctx) LIBVMAFContext *s = ctx->priv; int err = 0; - MetaStruct meta_data = { - .metadata = &s->metadata, - .comp = 0, - }; - VmafConfiguration cfg = { .log_level = log_level_map(av_log_get_level()), .n_subsample = s->n_subsample, .n_threads = s->n_threads, - .callback = set_meta, - .meta_data = &meta_data, - .meta_data_sz = sizeof(meta_data), }; err = vmaf_init(&s->vmaf, cfg); if (err) return AVERROR(EINVAL); + s->meta_data = malloc(sizeof(*s->meta_data)); + if (err) + return AVERROR(ENOMEM); + + s->meta_data->metadata = &s->metadata; + s->meta_data->comp = 0; + + VmafMetadataConfig meta_cfg = { + .callback = set_meta, + .data = s->meta_data, + .data_sz = sizeof(*s->meta_data), + }; + + err = vmaf_register_metadata_callback(s->vmaf, &meta_cfg); + if (err) { + return AVERROR(EINVAL); + } + err = parse_models(ctx); if (err) return err; From 1f864c79a9cff1e7f237a32a91370b7b55eaff65 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 22 Jul 2024 02:03:22 +0300 Subject: [PATCH 03/20] Remove data_sz according to libvmaf update --- libavfilter/vf_libvmaf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 790b88cfbf41e..91c5d74faabb9 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -475,7 +475,6 @@ static av_cold int init(AVFilterContext *ctx) VmafMetadataConfig meta_cfg = { .callback = set_meta, .data = s->meta_data, - .data_sz = sizeof(*s->meta_data), }; err = vmaf_register_metadata_callback(s->vmaf, &meta_cfg); From ec0c3c032beb0485801b7873f7e7902fbb63978d Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Tue, 30 Jul 2024 00:29:47 +0300 Subject: [PATCH 04/20] Update libvmaf according to support new changes --- libavfilter/vf_libvmaf.c | 58 +++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 91c5d74faabb9..d480d3f77be01 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -48,7 +48,6 @@ typedef struct MetaStruct { AVDictionary **metadata; - char comp; } MetaStruct; typedef struct LIBVMAFContext { @@ -112,23 +111,29 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } -//static void set_meta(AVDictionary **metadata, const char *key, char comp, float d) -static void set_meta(void *data, const char *key, double d) +static void dump_metadata(void *data, AVDictionary *metadata) { - if (!data) return; - MetaStruct *meta = data; - av_log(NULL, AV_LOG_ERROR, "key: %s, value: %f\n", key, d); - char value[128]; - snprintf(value, sizeof(value), "%f", d); - if (meta->comp) { - char key2[128]; - snprintf(key2, sizeof(key2), "%s%c", key, meta->comp); - av_dict_set(meta->metadata, key2, value, 0); - } else { - av_dict_set(meta->metadata, key, value, 0); + AVDictionaryEntry *tag = NULL; + MetaStruct *meta_data = data; + + while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { + av_log(NULL, AV_LOG_DEBUG, "VMAF feature: %s, score: %s\n", + tag->key, tag->value); + av_dict_set(meta_data->metadata, tag->key, tag->value, 0); } } +static void set_meta(void *data, VmafMetadata *metadata) +{ + MetaStruct *meta_data = data; + char value[128], key[128]; + snprintf(value, sizeof(value), "%0.2f", metadata->score); + snprintf(key, sizeof(key), "%s_%d", metadata->feature_name, metadata->picture_index); + av_dict_set(meta_data->metadata, key, value, 0); + av_log(NULL, AV_LOG_DEBUG, "VMAF feature: %s, score: %f\n", + key, metadata->score); +} + static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc) { const int bytes_per_value = bpc > 8 ? 2 : 1; @@ -470,14 +475,25 @@ static av_cold int init(AVFilterContext *ctx) return AVERROR(ENOMEM); s->meta_data->metadata = &s->metadata; - s->meta_data->comp = 0; - VmafMetadataConfig meta_cfg = { - .callback = set_meta, + VmafMetadataConfiguration meta_cfg = { + .feature_name = "vmaf", + .callback = &set_meta, .data = s->meta_data, }; - err = vmaf_register_metadata_callback(s->vmaf, &meta_cfg); + err = vmaf_register_metadata_callback(s->vmaf, meta_cfg); + if (err) { + return AVERROR(EINVAL); + } + + VmafMetadataConfiguration meta_cfg_1 = { + .feature_name = "psnr_y", + .callback = &set_meta, + .data = s->meta_data, + }; + + err = vmaf_register_metadata_callback(s->vmaf, meta_cfg_1); if (err) { return AVERROR(EINVAL); } @@ -606,6 +622,12 @@ static av_cold void uninit(AVFilterContext *ctx) "problem flushing libvmaf context.\n"); } + if (s->meta_data) { + av_log(ctx, AV_LOG_INFO, "DUMPING FEATURES\n"); + av_log(ctx, AV_LOG_INFO, "===============================\n\n"); + dump_metadata(s->meta_data, s->metadata); + } + for (unsigned i = 0; i < s->model_cnt; i++) { double vmaf_score; err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool), From 7b2b247073ad58df3ed93e6fdbb7e1b027fcfdc2 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Thu, 8 Aug 2024 13:57:40 +0300 Subject: [PATCH 05/20] Update vf_libvmaf according to new changes --- libavfilter/vf_libvmaf.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index d480d3f77be01..20ab047e3d914 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -117,7 +117,7 @@ static void dump_metadata(void *data, AVDictionary *metadata) MetaStruct *meta_data = data; while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { - av_log(NULL, AV_LOG_DEBUG, "VMAF feature: %s, score: %s\n", + av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %s\n", tag->key, tag->value); av_dict_set(meta_data->metadata, tag->key, tag->value, 0); } @@ -130,7 +130,7 @@ static void set_meta(void *data, VmafMetadata *metadata) snprintf(value, sizeof(value), "%0.2f", metadata->score); snprintf(key, sizeof(key), "%s_%d", metadata->feature_name, metadata->picture_index); av_dict_set(meta_data->metadata, key, value, 0); - av_log(NULL, AV_LOG_DEBUG, "VMAF feature: %s, score: %f\n", + av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); } @@ -482,7 +482,7 @@ static av_cold int init(AVFilterContext *ctx) .data = s->meta_data, }; - err = vmaf_register_metadata_callback(s->vmaf, meta_cfg); + err = vmaf_register_metadata_handler(s->vmaf, meta_cfg); if (err) { return AVERROR(EINVAL); } @@ -493,7 +493,7 @@ static av_cold int init(AVFilterContext *ctx) .data = s->meta_data, }; - err = vmaf_register_metadata_callback(s->vmaf, meta_cfg_1); + err = vmaf_register_metadata_handler(s->vmaf, meta_cfg_1); if (err) { return AVERROR(EINVAL); } From 6556a11140be4fb8c0a8e573ff1104e5efa0e469 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Fri, 9 Aug 2024 00:21:24 +0300 Subject: [PATCH 06/20] update vmaf filter according to new API changes (metadata propagation) --- libavfilter/vf_libvmaf.c | 139 +++++++++++++++++++++++++++++---------- 1 file changed, 106 insertions(+), 33 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 20ab047e3d914..37b0f43f531f0 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -60,8 +60,12 @@ typedef struct LIBVMAFContext { int n_subsample; char *model_cfg; char *feature_cfg; - AVDictionary *metadata; - MetaStruct *meta_data; + char *metadata_feature_cfg; + MetaStruct *metadata_priv; + struct { + VmafMetadataConfiguration *metadata_cfgs; + unsigned metadata_cfg_cnt; + } metadata_cfg_list; VmafContext *vmaf; VmafModel **model; unsigned model_cnt; @@ -83,6 +87,7 @@ static const AVOption libvmaf_options[] = { {"n_subsample", "Set interval for frame subsampling used when computing vmaf.", OFFSET(n_subsample), AV_OPT_TYPE_INT, {.i64=1}, 1, UINT_MAX, FLAGS}, {"model", "Set the model to be used for computing vmaf.", OFFSET(model_cfg), AV_OPT_TYPE_STRING, {.str="version=vmaf_v0.6.1"}, 0, 1, FLAGS}, {"feature", "Set the feature to be used for computing vmaf.", OFFSET(feature_cfg), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS}, + {"metadata_handler", "Set the feature to be propagated as metadata.", OFFSET(metadata_feature_cfg), AV_OPT_TYPE_STRING, {.str="name=vmaf"}, 0, 1, FLAGS}, { NULL } }; @@ -114,22 +119,22 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) static void dump_metadata(void *data, AVDictionary *metadata) { AVDictionaryEntry *tag = NULL; - MetaStruct *meta_data = data; + MetaStruct *metadata_priv = data; while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %s\n", tag->key, tag->value); - av_dict_set(meta_data->metadata, tag->key, tag->value, 0); + av_dict_set(metadata_priv->metadata, tag->key, tag->value, 0); } } static void set_meta(void *data, VmafMetadata *metadata) { - MetaStruct *meta_data = data; + MetaStruct *metadata_priv = data; char value[128], key[128]; snprintf(value, sizeof(value), "%0.2f", metadata->score); snprintf(key, sizeof(key), "%s_%d", metadata->feature_name, metadata->picture_index); - av_dict_set(meta_data->metadata, key, value, 0); + av_dict_set(metadata_priv->metadata, key, value, 0); av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); } @@ -437,6 +442,83 @@ static int parse_models(AVFilterContext *ctx) return err; } +static int parse_metadata_handlers(AVFilterContext *ctx) +{ + LIBVMAFContext *s = ctx->priv; + AVDictionary **dict; + unsigned dict_cnt; + int err = 0; + + if (!s->metadata_feature_cfg) + return 0; + + dict_cnt = 0; + dict = delimited_dict_parse(s->metadata_feature_cfg, &dict_cnt); + if (!dict) { + av_log(ctx, AV_LOG_ERROR, + "could not parse metadata feature config: %s\n", + s->metadata_feature_cfg); + return AVERROR(EINVAL); + } + + for (unsigned i = 0; i < dict_cnt; i++) { + VmafMetadataConfiguration *metadata_cfg = av_calloc(1, sizeof(*metadata_cfg)); + const AVDictionaryEntry *e = NULL; + char *feature_name = NULL; + + while (e = av_dict_iterate(dict[i], e)) { + if (!strcmp(e->key, "name")) { + metadata_cfg->feature_name = av_strdup(e->value); + continue; + } + } + + metadata_cfg->data = s->metadata_priv; + metadata_cfg->callback = &set_meta; + + err = vmaf_register_metadata_handler(s->vmaf, *metadata_cfg); + if (err) { + av_log(ctx, AV_LOG_ERROR, + "problem during vmaf_register_metadata_handler: %s\n", + feature_name); + goto exit; + } + + s->metadata_cfg_list.metadata_cfgs = av_realloc(s->metadata_cfg_list.metadata_cfgs, + (s->metadata_cfg_list.metadata_cfg_cnt + 1) * + sizeof(*s->metadata_cfg_list.metadata_cfgs)); + if (!s->metadata_cfg_list.metadata_cfgs) { + err = AVERROR(ENOMEM); + goto exit; + } + + s->metadata_cfg_list.metadata_cfgs[s->metadata_cfg_list.metadata_cfg_cnt++] = *metadata_cfg; + } + +exit: + for (unsigned i = 0; i < dict_cnt; i++) { + if (dict[i]) + av_dict_free(&dict[i]); + } + av_free(dict); + return err; +} + +static int init_metadata(AVFilterContext *ctx) +{ + LIBVMAFContext *s = ctx->priv; + + s->metadata_priv = av_calloc(1, sizeof(*s->metadata_priv)); + if (!s->metadata_priv) + return AVERROR(ENOMEM); + + s->metadata_priv->metadata = av_calloc(1, sizeof(AVDictionary*)); + if (!s->metadata_priv->metadata) + return AVERROR(ENOMEM); + + return 0; +} + static enum VmafLogLevel log_level_map(int log_level) { switch (log_level) { @@ -470,33 +552,13 @@ static av_cold int init(AVFilterContext *ctx) if (err) return AVERROR(EINVAL); - s->meta_data = malloc(sizeof(*s->meta_data)); + err = init_metadata(ctx); if (err) - return AVERROR(ENOMEM); - - s->meta_data->metadata = &s->metadata; - - VmafMetadataConfiguration meta_cfg = { - .feature_name = "vmaf", - .callback = &set_meta, - .data = s->meta_data, - }; - - err = vmaf_register_metadata_handler(s->vmaf, meta_cfg); - if (err) { - return AVERROR(EINVAL); - } - - VmafMetadataConfiguration meta_cfg_1 = { - .feature_name = "psnr_y", - .callback = &set_meta, - .data = s->meta_data, - }; + return err; - err = vmaf_register_metadata_handler(s->vmaf, meta_cfg_1); - if (err) { - return AVERROR(EINVAL); - } + err = parse_metadata_handlers(ctx); + if (err) + return err; err = parse_models(ctx); if (err) @@ -622,10 +684,21 @@ static av_cold void uninit(AVFilterContext *ctx) "problem flushing libvmaf context.\n"); } - if (s->meta_data) { + if (s->metadata_priv) { av_log(ctx, AV_LOG_INFO, "DUMPING FEATURES\n"); av_log(ctx, AV_LOG_INFO, "===============================\n\n"); - dump_metadata(s->meta_data, s->metadata); + dump_metadata(s->metadata_priv, *s->metadata_priv->metadata); + av_log(ctx, AV_LOG_INFO, "===============================\n\n"); + + av_dict_free(s->metadata_priv->metadata); + av_free(s->metadata_priv); + } + + if (s->metadata_cfg_list.metadata_cfgs) { + for (unsigned i = 0; i < s->metadata_cfg_list.metadata_cfg_cnt; i++) { + av_free(s->metadata_cfg_list.metadata_cfgs[i].feature_name); + } + av_free(s->metadata_cfg_list.metadata_cfgs); } for (unsigned i = 0; i < s->model_cnt; i++) { From 6944fff79dcc2a83a7aaf8d6ce448de1f0491bc9 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Wed, 21 Aug 2024 00:27:40 +0300 Subject: [PATCH 07/20] Remove metastruct --- libavfilter/vf_libvmaf.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 37b0f43f531f0..a642ee8cf8911 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -29,6 +29,7 @@ #include #include "libavutil/avstring.h" +#include "libavutil/dict.h" #include "libavutil/mem.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" @@ -46,10 +47,6 @@ #include "libavutil/hwcontext_cuda_internal.h" #endif -typedef struct MetaStruct { - AVDictionary **metadata; -} MetaStruct; - typedef struct LIBVMAFContext { const AVClass *class; FFFrameSync fs; @@ -61,7 +58,7 @@ typedef struct LIBVMAFContext { char *model_cfg; char *feature_cfg; char *metadata_feature_cfg; - MetaStruct *metadata_priv; + AVDictionary **metadata_priv; struct { VmafMetadataConfiguration *metadata_cfgs; unsigned metadata_cfg_cnt; @@ -119,22 +116,25 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) static void dump_metadata(void *data, AVDictionary *metadata) { AVDictionaryEntry *tag = NULL; - MetaStruct *metadata_priv = data; + AVDictionary **metadata_priv = data; while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %s\n", tag->key, tag->value); - av_dict_set(metadata_priv->metadata, tag->key, tag->value, 0); + av_dict_set(metadata_priv, tag->key, tag->value, 0); } } static void set_meta(void *data, VmafMetadata *metadata) { - MetaStruct *metadata_priv = data; + int err = 0; + AVDictionary **metadata_priv = data; char value[128], key[128]; snprintf(value, sizeof(value), "%0.2f", metadata->score); snprintf(key, sizeof(key), "%s_%d", metadata->feature_name, metadata->picture_index); - av_dict_set(metadata_priv->metadata, key, value, 0); + err = av_dict_set(metadata_priv, key, value, 0); + if (err < 0) + av_log(NULL, AV_LOG_ERROR, "could not set metadata: %s\n", key); av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); } @@ -508,14 +508,9 @@ static int init_metadata(AVFilterContext *ctx) { LIBVMAFContext *s = ctx->priv; - s->metadata_priv = av_calloc(1, sizeof(*s->metadata_priv)); + s->metadata_priv = av_calloc(1, sizeof(AVDictionary*)); if (!s->metadata_priv) return AVERROR(ENOMEM); - - s->metadata_priv->metadata = av_calloc(1, sizeof(AVDictionary*)); - if (!s->metadata_priv->metadata) - return AVERROR(ENOMEM); - return 0; } @@ -687,10 +682,10 @@ static av_cold void uninit(AVFilterContext *ctx) if (s->metadata_priv) { av_log(ctx, AV_LOG_INFO, "DUMPING FEATURES\n"); av_log(ctx, AV_LOG_INFO, "===============================\n\n"); - dump_metadata(s->metadata_priv, *s->metadata_priv->metadata); + dump_metadata(s->metadata_priv, *s->metadata_priv); av_log(ctx, AV_LOG_INFO, "===============================\n\n"); - av_dict_free(s->metadata_priv->metadata); + av_dict_free(s->metadata_priv); av_free(s->metadata_priv); } From aa7289e820409172e4000ab905b4a408118ec6dc Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Wed, 21 Aug 2024 21:17:22 +0300 Subject: [PATCH 08/20] Another Approach --- libavfilter/vf_libvmaf.c | 159 ++++++++++++++++++++++++++++++++------- 1 file changed, 132 insertions(+), 27 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index a642ee8cf8911..ab8e9299d7770 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -30,6 +30,7 @@ #include "libavutil/avstring.h" #include "libavutil/dict.h" +#include "libavutil/frame.h" #include "libavutil/mem.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" @@ -47,6 +48,13 @@ #include "libavutil/hwcontext_cuda_internal.h" #endif +typedef struct FrameList { + AVFrame *frame; + unsigned frame_number; + unsigned propagated_handlers_cnt; + struct FrameList *next; +} FrameList; + typedef struct LIBVMAFContext { const AVClass *class; FFFrameSync fs; @@ -58,11 +66,11 @@ typedef struct LIBVMAFContext { char *model_cfg; char *feature_cfg; char *metadata_feature_cfg; - AVDictionary **metadata_priv; struct { VmafMetadataConfiguration *metadata_cfgs; unsigned metadata_cfg_cnt; } metadata_cfg_list; + struct CallbackStruct *cb; VmafContext *vmaf; VmafModel **model; unsigned model_cnt; @@ -73,6 +81,11 @@ typedef struct LIBVMAFContext { #endif } LIBVMAFContext; +typedef struct CallbackStruct { + struct LIBVMAFContext *s; + FrameList *frame_list; +} CallbackStruct; + #define OFFSET(x) offsetof(LIBVMAFContext, x) #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM @@ -113,30 +126,116 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } -static void dump_metadata(void *data, AVDictionary *metadata) +static int add_to_frame_list(FrameList **head, AVFrame *frame, unsigned frame_number) +{ + FrameList *new_frame = av_malloc(sizeof(FrameList)); + if (!new_frame) + return AVERROR(ENOMEM); + + new_frame->frame = frame; + new_frame->frame_number = frame_number; + new_frame->propagated_handlers_cnt = 0; + new_frame->next = NULL; + + if (*head == NULL) { + *head = new_frame; + } else { + FrameList *current = *head; + while (current->next != NULL) { + current = current->next; + } + current->next = new_frame; + } + + return 0; +} + +static int remove_from_frame_list(FrameList **frame_list, unsigned frame_number) +{ + FrameList *cur = *frame_list; + FrameList *prev = NULL; + + while (cur) { + if (cur->frame_number == frame_number) { + if (prev) + prev->next = cur->next; + else + *frame_list = cur->next; + av_free(cur); + return 0; + } + prev = cur; + cur = cur->next; + } + + return AVERROR(EINVAL); +} + +static int free_frame_list(FrameList **frame_list) { - AVDictionaryEntry *tag = NULL; - AVDictionary **metadata_priv = data; + FrameList *cur = *frame_list; + while (cur) { + FrameList *next = cur->next; + av_frame_free(&cur->frame); + av_free(cur); + cur = next; + } + *frame_list = NULL; + return 0; +} - while ((tag = av_dict_get(metadata, "", tag, AV_DICT_IGNORE_SUFFIX))) { - av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %s\n", - tag->key, tag->value); - av_dict_set(metadata_priv, tag->key, tag->value, 0); +static FrameList* get_frame_from_frame_list(FrameList *frame_list, + unsigned frame_number) +{ + FrameList *cur = frame_list; + while (cur) { + if (cur->frame_number == frame_number) + return cur; + cur = cur->next; } + return NULL; } static void set_meta(void *data, VmafMetadata *metadata) { int err = 0; - AVDictionary **metadata_priv = data; + FrameList *current_frame = NULL; + CallbackStruct *cb = data; char value[128], key[128]; snprintf(value, sizeof(value), "%0.2f", metadata->score); - snprintf(key, sizeof(key), "%s_%d", metadata->feature_name, metadata->picture_index); - err = av_dict_set(metadata_priv, key, value, 0); + snprintf(key, sizeof(key), "%s.%d", metadata->feature_name, metadata->picture_index); + + current_frame = get_frame_from_frame_list(cb->frame_list, metadata->picture_index); + if (!current_frame) { + av_log(NULL, AV_LOG_ERROR, "could not find frame with index: %d\n", + metadata->picture_index); + return; + } + + err = av_dict_set(¤t_frame->frame->metadata, key, value, 0); if (err < 0) av_log(NULL, AV_LOG_ERROR, "could not set metadata: %s\n", key); - av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", - key, metadata->score); + + current_frame->propagated_handlers_cnt++; + + if (current_frame->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { + FrameList *cur = cb->frame_list; + // This code block allows to send frames monotonically + while(cur && cur->frame_number <= metadata->picture_index) { + if (cur->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { + FrameList *next; + // This necessary to avoid segfaults in filtergraphs + if (cb->s->fs.parent->outputs[0]) + ff_filter_frame(cb->s->fs.parent->outputs[0], cur->frame); + next = cur->next; + remove_from_frame_list(&cb->frame_list, cur->frame_number); + cur = next; + } + else + break; + } + } + av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); } static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc) @@ -194,13 +293,20 @@ static int do_vmaf(FFFrameSync *fs) return AVERROR(ENOMEM); } + err = add_to_frame_list(&s->cb->frame_list, dist, s->frame_cnt); + if (err) { + av_log(s, AV_LOG_ERROR, "problem during add_to_frame_list.\n"); + return AVERROR(ENOMEM); + } + err = vmaf_read_pictures(s->vmaf, &pic_ref, &pic_dist, s->frame_cnt++); if (err) { av_log(s, AV_LOG_ERROR, "problem during vmaf_read_pictures.\n"); return AVERROR(EINVAL); } - return ff_filter_frame(ctx->outputs[0], dist); + // It EXCEEDS the frame count, if dont use ff_filter_frame + return 0; } static AVDictionary **delimited_dict_parse(char *str, unsigned *cnt) @@ -473,7 +579,7 @@ static int parse_metadata_handlers(AVFilterContext *ctx) } } - metadata_cfg->data = s->metadata_priv; + metadata_cfg->data = s->cb; metadata_cfg->callback = &set_meta; err = vmaf_register_metadata_handler(s->vmaf, *metadata_cfg); @@ -508,9 +614,12 @@ static int init_metadata(AVFilterContext *ctx) { LIBVMAFContext *s = ctx->priv; - s->metadata_priv = av_calloc(1, sizeof(AVDictionary*)); - if (!s->metadata_priv) + s->cb = av_calloc(1, sizeof(CallbackStruct)); + if (!s->cb) return AVERROR(ENOMEM); + + s->cb->s = s; + return 0; } @@ -679,16 +788,6 @@ static av_cold void uninit(AVFilterContext *ctx) "problem flushing libvmaf context.\n"); } - if (s->metadata_priv) { - av_log(ctx, AV_LOG_INFO, "DUMPING FEATURES\n"); - av_log(ctx, AV_LOG_INFO, "===============================\n\n"); - dump_metadata(s->metadata_priv, *s->metadata_priv); - av_log(ctx, AV_LOG_INFO, "===============================\n\n"); - - av_dict_free(s->metadata_priv); - av_free(s->metadata_priv); - } - if (s->metadata_cfg_list.metadata_cfgs) { for (unsigned i = 0; i < s->metadata_cfg_list.metadata_cfg_cnt; i++) { av_free(s->metadata_cfg_list.metadata_cfgs[i].feature_name); @@ -713,6 +812,12 @@ static av_cold void uninit(AVFilterContext *ctx) vmaf_write_output(s->vmaf, s->log_path, log_fmt_map(s->log_fmt)); } + err = free_frame_list(&s->cb->frame_list); + if (err) { + av_log(ctx, AV_LOG_ERROR, + "problem freeing frame list.\n"); + } + clean_up: if (s->model) { for (unsigned i = 0; i < s->model_cnt; i++) { From 5855c9d0ad27c2c9c545b0b476e71ec04f75b31a Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Fri, 23 Aug 2024 04:20:13 +0300 Subject: [PATCH 09/20] Add macro guard for new api --- configure | 2 ++ libavfilter/allfilters.c | 1 + 2 files changed, 3 insertions(+) diff --git a/configure b/configure index 37178d7d81591..a807ca718e794 100755 --- a/configure +++ b/configure @@ -3958,6 +3958,7 @@ vflip_vulkan_filter_deps="vulkan spirv_compiler" vidstabdetect_filter_deps="libvidstab" vidstabtransform_filter_deps="libvidstab" libvmaf_filter_deps="libvmaf" +libvmaf_metadata_filter_deps="libvmaf libvmaf_metadata" libvmaf_cuda_filter_deps="libvmaf libvmaf_cuda ffnvcodec" zmq_filter_deps="libzmq" zoompan_filter_deps="swscale" @@ -6986,6 +6987,7 @@ enabled libv4l2 && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_i enabled libvidstab && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit enabled libvmaf && require_pkg_config libvmaf "libvmaf >= 2.0.0" libvmaf.h vmaf_init enabled libvmaf && check_pkg_config libvmaf_cuda "libvmaf >= 2.0.0" libvmaf_cuda.h vmaf_cuda_state_init +enabled libvmaf && check_pkg_config libvmaf_metadata "libvmaf >= 2.0.0" libvmaf.h vmaf_register_metadata_handler enabled libvo_amrwbenc && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_init -lvo-amrwbenc enabled libvorbis && require_pkg_config libvorbis vorbis vorbis/codec.h vorbis_info_init && require_pkg_config libvorbisenc vorbisenc vorbis/vorbisenc.h vorbis_encode_init diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 63600e9b58cfe..b8f862ba5f222 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -342,6 +342,7 @@ extern const AVFilter ff_vf_lensfun; extern const AVFilter ff_vf_libplacebo; extern const AVFilter ff_vf_libvmaf; extern const AVFilter ff_vf_libvmaf_cuda; +extern const AVFilter ff_vf_libvmaf_metadata; extern const AVFilter ff_vf_limitdiff; extern const AVFilter ff_vf_limiter; extern const AVFilter ff_vf_loop; From 35f34a00187e45296b77eefc26daf9a87af4d4e2 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Fri, 23 Aug 2024 18:10:10 +0300 Subject: [PATCH 10/20] Add, defined macro guardsto vf_libvmaf --- libavfilter/vf_libvmaf.c | 59 +++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index ab8e9299d7770..032d12a631c69 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -48,6 +48,7 @@ #include "libavutil/hwcontext_cuda_internal.h" #endif +#if CONFIG_LIBVMAF_METADATA_FILTER typedef struct FrameList { AVFrame *frame; unsigned frame_number; @@ -55,6 +56,12 @@ typedef struct FrameList { struct FrameList *next; } FrameList; +typedef struct CallbackStruct { + struct LIBVMAFContext *s; + FrameList *frame_list; +} CallbackStruct; +#endif + typedef struct LIBVMAFContext { const AVClass *class; FFFrameSync fs; @@ -64,13 +71,15 @@ typedef struct LIBVMAFContext { int n_threads; int n_subsample; char *model_cfg; +#if CONFIG_LIBVMAF_METADATA_FILTER char *feature_cfg; char *metadata_feature_cfg; struct { VmafMetadataConfiguration *metadata_cfgs; unsigned metadata_cfg_cnt; } metadata_cfg_list; - struct CallbackStruct *cb; + CallbackStruct *cb; +#endif VmafContext *vmaf; VmafModel **model; unsigned model_cnt; @@ -81,11 +90,6 @@ typedef struct LIBVMAFContext { #endif } LIBVMAFContext; -typedef struct CallbackStruct { - struct LIBVMAFContext *s; - FrameList *frame_list; -} CallbackStruct; - #define OFFSET(x) offsetof(LIBVMAFContext, x) #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM @@ -126,6 +130,7 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } +#if CONFIG_LIBVMAF_METADATA_FILTER static int add_to_frame_list(FrameList **head, AVFrame *frame, unsigned frame_number) { FrameList *new_frame = av_malloc(sizeof(FrameList)); @@ -237,6 +242,7 @@ static void set_meta(void *data, VmafMetadata *metadata) } av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); } +#endif static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc) { @@ -293,11 +299,13 @@ static int do_vmaf(FFFrameSync *fs) return AVERROR(ENOMEM); } +#if CONFIG_LIBVMAF_METADATA_FILTER err = add_to_frame_list(&s->cb->frame_list, dist, s->frame_cnt); if (err) { av_log(s, AV_LOG_ERROR, "problem during add_to_frame_list.\n"); return AVERROR(ENOMEM); } +#endif err = vmaf_read_pictures(s->vmaf, &pic_ref, &pic_dist, s->frame_cnt++); if (err) { @@ -305,8 +313,11 @@ static int do_vmaf(FFFrameSync *fs) return AVERROR(EINVAL); } - // It EXCEEDS the frame count, if dont use ff_filter_frame +#if CONFIG_LIBVMAF_METADATA_FILTER return 0; +#else + return ff_filter_frame(ctx->outputs[0], dist); +#endif } static AVDictionary **delimited_dict_parse(char *str, unsigned *cnt) @@ -548,6 +559,7 @@ static int parse_models(AVFilterContext *ctx) return err; } +#if CONFIG_LIBVMAF_METADATA_FILTER static int parse_metadata_handlers(AVFilterContext *ctx) { LIBVMAFContext *s = ctx->priv; @@ -622,6 +634,7 @@ static int init_metadata(AVFilterContext *ctx) return 0; } +#endif static enum VmafLogLevel log_level_map(int log_level) { @@ -656,6 +669,7 @@ static av_cold int init(AVFilterContext *ctx) if (err) return AVERROR(EINVAL); +#if CONFIG_LIBVMAF_METADATA_FILTER err = init_metadata(ctx); if (err) return err; @@ -663,6 +677,7 @@ static av_cold int init(AVFilterContext *ctx) err = parse_metadata_handlers(ctx); if (err) return err; +#endif err = parse_models(ctx); if (err) @@ -788,6 +803,7 @@ static av_cold void uninit(AVFilterContext *ctx) "problem flushing libvmaf context.\n"); } +#if CONFIG_LIBVMAF_METADATA_FILTER if (s->metadata_cfg_list.metadata_cfgs) { for (unsigned i = 0; i < s->metadata_cfg_list.metadata_cfg_cnt; i++) { av_free(s->metadata_cfg_list.metadata_cfgs[i].feature_name); @@ -795,6 +811,13 @@ static av_cold void uninit(AVFilterContext *ctx) av_free(s->metadata_cfg_list.metadata_cfgs); } + err = free_frame_list(&s->cb->frame_list); + if (err) { + av_log(ctx, AV_LOG_ERROR, + "problem freeing frame list.\n"); + } +#endif + for (unsigned i = 0; i < s->model_cnt; i++) { double vmaf_score; err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool), @@ -812,12 +835,6 @@ static av_cold void uninit(AVFilterContext *ctx) vmaf_write_output(s->vmaf, s->log_path, log_fmt_map(s->log_fmt)); } - err = free_frame_list(&s->cb->frame_list); - if (err) { - av_log(ctx, AV_LOG_ERROR, - "problem freeing frame list.\n"); - } - clean_up: if (s->model) { for (unsigned i = 0; i < s->model_cnt; i++) { @@ -1058,3 +1075,19 @@ const AVFilter ff_vf_libvmaf_cuda = { .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, }; #endif + +#if CONFIG_LIBVMAF_METADATA_FILTER +const AVFilter ff_vf_libvmaf_metadata = { + .name = "libvmaf_cuda", + .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."), + .preinit = libvmaf_framesync_preinit, + .init = init, + .uninit = uninit, + .activate = activate, + .priv_size = sizeof(LIBVMAFContext), + .priv_class = &libvmaf_class, + FILTER_INPUTS(libvmaf_inputs), + FILTER_OUTPUTS(libvmaf_outputs), + FILTER_PIXFMTS_ARRAY(pix_fmts), +}; +#endif From a657cf376e48248ff9afb49a05710e1ae3019ca4 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sat, 24 Aug 2024 00:45:31 +0300 Subject: [PATCH 11/20] Some bugfix and compilation error fix --- libavfilter/vf_libvmaf.c | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 032d12a631c69..870cb68a743f4 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -71,8 +71,8 @@ typedef struct LIBVMAFContext { int n_threads; int n_subsample; char *model_cfg; -#if CONFIG_LIBVMAF_METADATA_FILTER char *feature_cfg; +#if CONFIG_LIBVMAF_METADATA_FILTER char *metadata_feature_cfg; struct { VmafMetadataConfiguration *metadata_cfgs; @@ -90,6 +90,7 @@ typedef struct LIBVMAFContext { #endif } LIBVMAFContext; +#define MAX_PENDING_FRAMES 24 #define OFFSET(x) offsetof(LIBVMAFContext, x) #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM @@ -101,7 +102,9 @@ static const AVOption libvmaf_options[] = { {"n_subsample", "Set interval for frame subsampling used when computing vmaf.", OFFSET(n_subsample), AV_OPT_TYPE_INT, {.i64=1}, 1, UINT_MAX, FLAGS}, {"model", "Set the model to be used for computing vmaf.", OFFSET(model_cfg), AV_OPT_TYPE_STRING, {.str="version=vmaf_v0.6.1"}, 0, 1, FLAGS}, {"feature", "Set the feature to be used for computing vmaf.", OFFSET(feature_cfg), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS}, +#if CONFIG_LIBVMAF_METADATA_FILTER {"metadata_handler", "Set the feature to be propagated as metadata.", OFFSET(metadata_feature_cfg), AV_OPT_TYPE_STRING, {.str="name=vmaf"}, 0, 1, FLAGS}, +#endif { NULL } }; @@ -230,8 +233,11 @@ static void set_meta(void *data, VmafMetadata *metadata) if (cur->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { FrameList *next; // This necessary to avoid segfaults in filtergraphs - if (cb->s->fs.parent->outputs[0]) + if (cb->s->fs.parent->outputs[0]) { ff_filter_frame(cb->s->fs.parent->outputs[0], cur->frame); + } + else + av_frame_free(&cur->frame); next = cur->next; remove_from_frame_list(&cb->frame_list, cur->frame_number); cur = next; @@ -240,7 +246,7 @@ static void set_meta(void *data, VmafMetadata *metadata) break; } } - av_log(NULL, AV_LOG_INFO, "VMAF feature: %s, score: %f\n", key, metadata->score); + av_log(NULL, AV_LOG_WARNING, "VMAF feature: %s, score: %f\n", key, metadata->score); } #endif @@ -272,8 +278,9 @@ static int do_vmaf(FFFrameSync *fs) VmafPicture pic_ref, pic_dist; AVFrame *ref, *dist; int err = 0; + int ret = 0; - int ret = ff_framesync_dualinput_get(fs, &dist, &ref); + ret = ff_framesync_dualinput_get(fs, &dist, &ref); if (ret < 0) return ret; if (ctx->is_disabled || !ref) @@ -314,7 +321,10 @@ static int do_vmaf(FFFrameSync *fs) } #if CONFIG_LIBVMAF_METADATA_FILTER - return 0; + if (!s->metadata_cfg_list.metadata_cfg_cnt) + return ff_filter_frame(ctx->outputs[0], dist); + else + return 0; #else return ff_filter_frame(ctx->outputs[0], dist); #endif @@ -1076,9 +1086,8 @@ const AVFilter ff_vf_libvmaf_cuda = { }; #endif -#if CONFIG_LIBVMAF_METADATA_FILTER const AVFilter ff_vf_libvmaf_metadata = { - .name = "libvmaf_cuda", + .name = "libvmaf_metadata", .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."), .preinit = libvmaf_framesync_preinit, .init = init, @@ -1090,4 +1099,3 @@ const AVFilter ff_vf_libvmaf_metadata = { FILTER_OUTPUTS(libvmaf_outputs), FILTER_PIXFMTS_ARRAY(pix_fmts), }; -#endif From dfbf38414761956e543766234246509d1c35eb7a Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sat, 24 Aug 2024 02:35:20 +0300 Subject: [PATCH 12/20] remove unrelated change --- libavfilter/vf_libvmaf.c | 1 - 1 file changed, 1 deletion(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 870cb68a743f4..def0e63efd8ae 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -90,7 +90,6 @@ typedef struct LIBVMAFContext { #endif } LIBVMAFContext; -#define MAX_PENDING_FRAMES 24 #define OFFSET(x) offsetof(LIBVMAFContext, x) #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM From 3ad9e15640089d3fef72089e986559faa9197382 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sat, 24 Aug 2024 02:57:35 +0300 Subject: [PATCH 13/20] update guard macros with using vmaf version --- configure | 2 -- libavfilter/allfilters.c | 1 - libavfilter/vf_libvmaf.c | 27 +++++++++++---------------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/configure b/configure index a807ca718e794..37178d7d81591 100755 --- a/configure +++ b/configure @@ -3958,7 +3958,6 @@ vflip_vulkan_filter_deps="vulkan spirv_compiler" vidstabdetect_filter_deps="libvidstab" vidstabtransform_filter_deps="libvidstab" libvmaf_filter_deps="libvmaf" -libvmaf_metadata_filter_deps="libvmaf libvmaf_metadata" libvmaf_cuda_filter_deps="libvmaf libvmaf_cuda ffnvcodec" zmq_filter_deps="libzmq" zoompan_filter_deps="swscale" @@ -6987,7 +6986,6 @@ enabled libv4l2 && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_i enabled libvidstab && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit enabled libvmaf && require_pkg_config libvmaf "libvmaf >= 2.0.0" libvmaf.h vmaf_init enabled libvmaf && check_pkg_config libvmaf_cuda "libvmaf >= 2.0.0" libvmaf_cuda.h vmaf_cuda_state_init -enabled libvmaf && check_pkg_config libvmaf_metadata "libvmaf >= 2.0.0" libvmaf.h vmaf_register_metadata_handler enabled libvo_amrwbenc && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_init -lvo-amrwbenc enabled libvorbis && require_pkg_config libvorbis vorbis vorbis/codec.h vorbis_info_init && require_pkg_config libvorbisenc vorbisenc vorbis/vorbisenc.h vorbis_encode_init diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index b8f862ba5f222..63600e9b58cfe 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -342,7 +342,6 @@ extern const AVFilter ff_vf_lensfun; extern const AVFilter ff_vf_libplacebo; extern const AVFilter ff_vf_libvmaf; extern const AVFilter ff_vf_libvmaf_cuda; -extern const AVFilter ff_vf_libvmaf_metadata; extern const AVFilter ff_vf_limitdiff; extern const AVFilter ff_vf_limiter; extern const AVFilter ff_vf_loop; diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index def0e63efd8ae..21ee786cab4fb 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -27,6 +27,7 @@ #include "config_components.h" #include +#include #include "libavutil/avstring.h" #include "libavutil/dict.h" @@ -48,6 +49,15 @@ #include "libavutil/hwcontext_cuda_internal.h" #endif +#define VMAF_VERSION_INT_VER(major, minor, patch) \ + ((major) * 10000 + (minor) * 100 + (patch)) + +#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(2, 0, 0) +#define CONFIG_LIBVMAF_METADATA_FILTER 1 +#else +#define CONFIG_LIBVMAF_METADATA_FILTER 0 +#endif + #if CONFIG_LIBVMAF_METADATA_FILTER typedef struct FrameList { AVFrame *frame; @@ -277,9 +287,8 @@ static int do_vmaf(FFFrameSync *fs) VmafPicture pic_ref, pic_dist; AVFrame *ref, *dist; int err = 0; - int ret = 0; - ret = ff_framesync_dualinput_get(fs, &dist, &ref); + int ret = ff_framesync_dualinput_get(fs, &dist, &ref); if (ret < 0) return ret; if (ctx->is_disabled || !ref) @@ -1084,17 +1093,3 @@ const AVFilter ff_vf_libvmaf_cuda = { .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, }; #endif - -const AVFilter ff_vf_libvmaf_metadata = { - .name = "libvmaf_metadata", - .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."), - .preinit = libvmaf_framesync_preinit, - .init = init, - .uninit = uninit, - .activate = activate, - .priv_size = sizeof(LIBVMAFContext), - .priv_class = &libvmaf_class, - FILTER_INPUTS(libvmaf_inputs), - FILTER_OUTPUTS(libvmaf_outputs), - FILTER_PIXFMTS_ARRAY(pix_fmts), -}; From 364b2194b511ad8a1f2a6597f04df5188fbf2d02 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sat, 24 Aug 2024 02:59:31 +0300 Subject: [PATCH 14/20] Update vmaf version to 2.0.0 to 3.0.0 in the filter --- libavfilter/vf_libvmaf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 21ee786cab4fb..45b2e81e5dc8a 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -52,7 +52,7 @@ #define VMAF_VERSION_INT_VER(major, minor, patch) \ ((major) * 10000 + (minor) * 100 + (patch)) -#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(2, 0, 0) +#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(3, 0, 0) #define CONFIG_LIBVMAF_METADATA_FILTER 1 #else #define CONFIG_LIBVMAF_METADATA_FILTER 0 From e324fc9c2b20b4842ee846dc3bbad74336383b74 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 26 Aug 2024 17:58:32 +0300 Subject: [PATCH 15/20] Fixed sync issues --- libavfilter/vf_libvmaf.c | 124 +++++++++++++++++++++++++++++++-------- 1 file changed, 99 insertions(+), 25 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 45b2e81e5dc8a..1050c997e79f7 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -52,13 +52,15 @@ #define VMAF_VERSION_INT_VER(major, minor, patch) \ ((major) * 10000 + (minor) * 100 + (patch)) -#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(3, 0, 0) -#define CONFIG_LIBVMAF_METADATA_FILTER 1 -#else -#define CONFIG_LIBVMAF_METADATA_FILTER 0 +#if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(2, 0, 0) +#define CONFIG_LIBVMAF_METADATA_ENABLED 1 +#else +#define CONFIG_LIBVMAF_METADATA_ENABLED 0 #endif -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED +#include + typedef struct FrameList { AVFrame *frame; unsigned frame_number; @@ -82,16 +84,19 @@ typedef struct LIBVMAFContext { int n_subsample; char *model_cfg; char *feature_cfg; -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED char *metadata_feature_cfg; struct { VmafMetadataConfiguration *metadata_cfgs; unsigned metadata_cfg_cnt; } metadata_cfg_list; CallbackStruct *cb; + atomic_uint outlink_eof; + atomic_uint eof_frame; #endif VmafContext *vmaf; VmafModel **model; + int flushed; unsigned model_cnt; unsigned frame_cnt; unsigned bpc; @@ -111,7 +116,7 @@ static const AVOption libvmaf_options[] = { {"n_subsample", "Set interval for frame subsampling used when computing vmaf.", OFFSET(n_subsample), AV_OPT_TYPE_INT, {.i64=1}, 1, UINT_MAX, FLAGS}, {"model", "Set the model to be used for computing vmaf.", OFFSET(model_cfg), AV_OPT_TYPE_STRING, {.str="version=vmaf_v0.6.1"}, 0, 1, FLAGS}, {"feature", "Set the feature to be used for computing vmaf.", OFFSET(feature_cfg), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS}, -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED {"metadata_handler", "Set the feature to be propagated as metadata.", OFFSET(metadata_feature_cfg), AV_OPT_TYPE_STRING, {.str="name=vmaf"}, 0, 1, FLAGS}, #endif { NULL } @@ -142,7 +147,7 @@ static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt) } } -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED static int add_to_frame_list(FrameList **head, AVFrame *frame, unsigned frame_number) { FrameList *new_frame = av_malloc(sizeof(FrameList)); @@ -241,12 +246,14 @@ static void set_meta(void *data, VmafMetadata *metadata) while(cur && cur->frame_number <= metadata->picture_index) { if (cur->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { FrameList *next; - // This necessary to avoid segfaults in filtergraphs - if (cb->s->fs.parent->outputs[0]) { + av_log(cb->s->fs.parent, AV_LOG_DEBUG, "eof %d \n", cb->s->outlink_eof); + // Check outlink is closed + av_log(cb->s->fs.parent, AV_LOG_DEBUG, "RETTTTTTTTt %d\n", cb->s->eof_frame <= cb->s->frame_cnt - 1); + if (cb->s->eof_frame <= cb->s->frame_cnt - 1 && !cb->s->outlink_eof) { + av_log(cb->s->fs.parent, AV_LOG_DEBUG, "VMAF feature: %d, score: %f\n", cur->frame_number, metadata->score); + cb->s->eof_frame = cur->frame_number; ff_filter_frame(cb->s->fs.parent->outputs[0], cur->frame); } - else - av_frame_free(&cur->frame); next = cur->next; remove_from_frame_list(&cb->frame_list, cur->frame_number); cur = next; @@ -255,7 +262,6 @@ static void set_meta(void *data, VmafMetadata *metadata) break; } } - av_log(NULL, AV_LOG_WARNING, "VMAF feature: %s, score: %f\n", key, metadata->score); } #endif @@ -314,7 +320,7 @@ static int do_vmaf(FFFrameSync *fs) return AVERROR(ENOMEM); } -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED err = add_to_frame_list(&s->cb->frame_list, dist, s->frame_cnt); if (err) { av_log(s, AV_LOG_ERROR, "problem during add_to_frame_list.\n"); @@ -328,11 +334,13 @@ static int do_vmaf(FFFrameSync *fs) return AVERROR(EINVAL); } -#if CONFIG_LIBVMAF_METADATA_FILTER - if (!s->metadata_cfg_list.metadata_cfg_cnt) - return ff_filter_frame(ctx->outputs[0], dist); - else +#if CONFIG_LIBVMAF_METADATA_ENABLED + if (s->metadata_cfg_list.metadata_cfg_cnt) { + av_log(s, AV_LOG_DEBUG, "frame: %d\n", s->frame_cnt - 1); return 0; + } + else + return ff_filter_frame(ctx->outputs[0], dist); #else return ff_filter_frame(ctx->outputs[0], dist); #endif @@ -577,7 +585,7 @@ static int parse_models(AVFilterContext *ctx) return err; } -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED static int parse_metadata_handlers(AVFilterContext *ctx) { LIBVMAFContext *s = ctx->priv; @@ -687,7 +695,7 @@ static av_cold int init(AVFilterContext *ctx) if (err) return AVERROR(EINVAL); -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED err = init_metadata(ctx); if (err) return err; @@ -772,6 +780,54 @@ static int config_output(AVFilterLink *outlink) static int activate(AVFilterContext *ctx) { LIBVMAFContext *s = ctx->priv; +#if CONFIG_LIBVMAF_METADATA_ENABLED + // There are 2 cases for metadata propagation: + // 1. Where the case that outlink closes + // 2. Where inlink closes + // Case 1: + // In this case we need check outlink somehow for the status in every iteration. + // If outlink is not wanting frame anymore, we need to proceed with uninit with setting inlink. + // But nature of multithreading settting eof inside the activate call can make sync issues and + // can lead to extra propagated frames. Atomic variables are used to avoid this. + // Case 2: + // This case relatively easy to handle. Because of calculation of vmaf score takes time + // So `do_vmaf` buffers many of frames before sending to outlink that causes + // premature close of outlink. + // Checking inlink status is enough and if inlinkeof flushing vmaf is enough for this. + int64_t pts; + int status, ret = 0; + ret = ff_outlink_get_status(ctx->outputs[0]); + av_log(ctx, AV_LOG_DEBUG, "outlink ret: %d\n", ret); + if (ret){ + av_log(ctx, AV_LOG_DEBUG, "FRAME NOT WANTED \n" ); + s->outlink_eof = 1; + ff_inlink_set_status(ctx->inputs[0], ret); + return 0; + } + if (s->eof_frame <= s->frame_cnt - 1) { + av_log(ctx, AV_LOG_DEBUG, "EOF FRAME: %d\n", s->eof_frame); + } + ret = ff_inlink_acknowledge_status(ctx->inputs[0], &status, &pts); + av_log(ctx, AV_LOG_DEBUG, "inlink ret: %d status %d \n", ret, status); + av_log(s, AV_LOG_DEBUG, "activate frame cnt: %d\n", s->frame_cnt - 1); + if (ret){ + // Make sure last frame is processed + ff_framesync_activate(&s->fs); + + av_log(ctx, AV_LOG_DEBUG, "starting flush\n"); + ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); + if (ret) { + av_log(ctx, AV_LOG_ERROR, + "problem flushing libvmaf context.\n"); + } + s->flushed = 1; + av_log(ctx, AV_LOG_DEBUG, "vmaf flushed\n"); + + ff_outlink_set_status(ctx->outputs[0], status, pts); + av_log(ctx, AV_LOG_DEBUG, "outlink status set\n"); + return 0; + } +#endif return ff_framesync_activate(&s->fs); } @@ -810,18 +866,28 @@ static av_cold void uninit(AVFilterContext *ctx) LIBVMAFContext *s = ctx->priv; int err = 0; + av_log(ctx, AV_LOG_DEBUG, "UNINIT\n\n\n\n\n\n\n"); + + if (!s->outlink_eof) { + s->outlink_eof = 1; + av_log(ctx, AV_LOG_DEBUG, "EOF SET\n"); + } + ff_framesync_uninit(&s->fs); if (!s->frame_cnt) goto clean_up; - err = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); - if (err) { - av_log(ctx, AV_LOG_ERROR, - "problem flushing libvmaf context.\n"); + if (!s->flushed) { + err = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); + if (err) { + av_log(ctx, AV_LOG_ERROR, + "problem flushing libvmaf context.\n"); + } + s->flushed = 1; } -#if CONFIG_LIBVMAF_METADATA_FILTER +#if CONFIG_LIBVMAF_METADATA_ENABLED if (s->metadata_cfg_list.metadata_cfgs) { for (unsigned i = 0; i < s->metadata_cfg_list.metadata_cfg_cnt; i++) { av_free(s->metadata_cfg_list.metadata_cfgs[i].feature_name); @@ -838,8 +904,16 @@ static av_cold void uninit(AVFilterContext *ctx) for (unsigned i = 0; i < s->model_cnt; i++) { double vmaf_score; + +#if CONFIG_LIBVMAF_METADATA_ENABLED + err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool), + &vmaf_score, 0, s->eof_frame); + av_log(ctx, AV_LOG_DEBUG, "frame: %d frame_cnt %d\n", s->eof_frame, s->frame_cnt - 1); +#else err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool), &vmaf_score, 0, s->frame_cnt - 1); +#endif + if (err) { av_log(ctx, AV_LOG_ERROR, "problem getting pooled vmaf score.\n"); From 7a22076464202611d97b7967deaf511e49fb2215 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 26 Aug 2024 18:01:09 +0300 Subject: [PATCH 16/20] Removed unnecessary logs --- libavfilter/vf_libvmaf.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 1050c997e79f7..b24b9897fcc9a 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -246,9 +246,7 @@ static void set_meta(void *data, VmafMetadata *metadata) while(cur && cur->frame_number <= metadata->picture_index) { if (cur->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { FrameList *next; - av_log(cb->s->fs.parent, AV_LOG_DEBUG, "eof %d \n", cb->s->outlink_eof); // Check outlink is closed - av_log(cb->s->fs.parent, AV_LOG_DEBUG, "RETTTTTTTTt %d\n", cb->s->eof_frame <= cb->s->frame_cnt - 1); if (cb->s->eof_frame <= cb->s->frame_cnt - 1 && !cb->s->outlink_eof) { av_log(cb->s->fs.parent, AV_LOG_DEBUG, "VMAF feature: %d, score: %f\n", cur->frame_number, metadata->score); cb->s->eof_frame = cur->frame_number; @@ -797,34 +795,24 @@ static int activate(AVFilterContext *ctx) int64_t pts; int status, ret = 0; ret = ff_outlink_get_status(ctx->outputs[0]); - av_log(ctx, AV_LOG_DEBUG, "outlink ret: %d\n", ret); if (ret){ - av_log(ctx, AV_LOG_DEBUG, "FRAME NOT WANTED \n" ); s->outlink_eof = 1; ff_inlink_set_status(ctx->inputs[0], ret); return 0; } - if (s->eof_frame <= s->frame_cnt - 1) { - av_log(ctx, AV_LOG_DEBUG, "EOF FRAME: %d\n", s->eof_frame); - } ret = ff_inlink_acknowledge_status(ctx->inputs[0], &status, &pts); - av_log(ctx, AV_LOG_DEBUG, "inlink ret: %d status %d \n", ret, status); - av_log(s, AV_LOG_DEBUG, "activate frame cnt: %d\n", s->frame_cnt - 1); if (ret){ // Make sure last frame is processed ff_framesync_activate(&s->fs); - av_log(ctx, AV_LOG_DEBUG, "starting flush\n"); ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); if (ret) { av_log(ctx, AV_LOG_ERROR, "problem flushing libvmaf context.\n"); } s->flushed = 1; - av_log(ctx, AV_LOG_DEBUG, "vmaf flushed\n"); ff_outlink_set_status(ctx->outputs[0], status, pts); - av_log(ctx, AV_LOG_DEBUG, "outlink status set\n"); return 0; } #endif @@ -866,12 +854,8 @@ static av_cold void uninit(AVFilterContext *ctx) LIBVMAFContext *s = ctx->priv; int err = 0; - av_log(ctx, AV_LOG_DEBUG, "UNINIT\n\n\n\n\n\n\n"); - - if (!s->outlink_eof) { + if (!s->outlink_eof) s->outlink_eof = 1; - av_log(ctx, AV_LOG_DEBUG, "EOF SET\n"); - } ff_framesync_uninit(&s->fs); From 726702d8166c0e04f926b2388e24af803b64713d Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 26 Aug 2024 18:09:53 +0300 Subject: [PATCH 17/20] add filters.h --- libavfilter/vf_libvmaf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index b24b9897fcc9a..5a2ad181f836b 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -36,6 +36,7 @@ #include "libavutil/opt.h" #include "libavutil/pixdesc.h" #include "avfilter.h" +#include "filters.h" #include "drawutils.h" #include "formats.h" #include "framesync.h" From facfbb3328d8b0b646edc8b91447388d94ceb41d Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 26 Aug 2024 18:54:22 +0300 Subject: [PATCH 18/20] Fixed build issues and made some optimizations --- libavfilter/vf_libvmaf.c | 46 ++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 5a2ad181f836b..8b207fe4499f0 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -248,10 +248,11 @@ static void set_meta(void *data, VmafMetadata *metadata) if (cur->propagated_handlers_cnt == cb->s->metadata_cfg_list.metadata_cfg_cnt) { FrameList *next; // Check outlink is closed - if (cb->s->eof_frame <= cb->s->frame_cnt - 1 && !cb->s->outlink_eof) { + if (!cb->s->outlink_eof) { av_log(cb->s->fs.parent, AV_LOG_DEBUG, "VMAF feature: %d, score: %f\n", cur->frame_number, metadata->score); cb->s->eof_frame = cur->frame_number; - ff_filter_frame(cb->s->fs.parent->outputs[0], cur->frame); + if(ff_filter_frame(cb->s->fs.parent->outputs[0], cur->frame)) + return; } next = cur->next; remove_from_frame_list(&cb->frame_list, cur->frame_number); @@ -334,10 +335,8 @@ static int do_vmaf(FFFrameSync *fs) } #if CONFIG_LIBVMAF_METADATA_ENABLED - if (s->metadata_cfg_list.metadata_cfg_cnt) { - av_log(s, AV_LOG_DEBUG, "frame: %d\n", s->frame_cnt - 1); + if (s->metadata_cfg_list.metadata_cfg_cnt) return 0; - } else return ff_filter_frame(ctx->outputs[0], dist); #else @@ -792,29 +791,24 @@ static int activate(AVFilterContext *ctx) // This case relatively easy to handle. Because of calculation of vmaf score takes time // So `do_vmaf` buffers many of frames before sending to outlink that causes // premature close of outlink. - // Checking inlink status is enough and if inlinkeof flushing vmaf is enough for this. + // Checking inlink status is enough and if inlink == eof flushing vmaf is enough for this. int64_t pts; int status, ret = 0; - ret = ff_outlink_get_status(ctx->outputs[0]); - if (ret){ + + if (ff_outlink_get_status(ctx->outputs[0])) s->outlink_eof = 1; - ff_inlink_set_status(ctx->inputs[0], ret); - return 0; - } - ret = ff_inlink_acknowledge_status(ctx->inputs[0], &status, &pts); - if (ret){ - // Make sure last frame is processed - ff_framesync_activate(&s->fs); - ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); - if (ret) { - av_log(ctx, AV_LOG_ERROR, - "problem flushing libvmaf context.\n"); + for (unsigned i = 0; i < ctx->nb_inputs; i++) { + if (ff_inlink_acknowledge_status(ctx->inputs[i], &status, &pts)){ + if (!s->flushed) { + ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); + if (ret) + av_log(ctx, AV_LOG_ERROR, + "problem flushing libvmaf context.\n"); + else + s->flushed = 1; + } } - s->flushed = 1; - - ff_outlink_set_status(ctx->outputs[0], status, pts); - return 0; } #endif return ff_framesync_activate(&s->fs); @@ -855,8 +849,10 @@ static av_cold void uninit(AVFilterContext *ctx) LIBVMAFContext *s = ctx->priv; int err = 0; +#if CONFIG_LIBVMAF_METADATA_ENABLED if (!s->outlink_eof) s->outlink_eof = 1; +#endif ff_framesync_uninit(&s->fs); @@ -868,8 +864,8 @@ static av_cold void uninit(AVFilterContext *ctx) if (err) { av_log(ctx, AV_LOG_ERROR, "problem flushing libvmaf context.\n"); - } - s->flushed = 1; + } else + s->flushed = 1; } #if CONFIG_LIBVMAF_METADATA_ENABLED From f8b66bbe4b425197b53f965b90b73b1e410107a4 Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Sun, 30 Jun 2024 02:23:06 +0300 Subject: [PATCH 19/20] merge with other branch --- libavfilter/vf_libvmaf.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 8b207fe4499f0..4c5dedc5df5fe 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -54,6 +54,7 @@ ((major) * 10000 + (minor) * 100 + (patch)) #if VMAF_VERSION_INT_VER(VMAF_API_VERSION_MAJOR, VMAF_API_VERSION_MINOR, VMAF_API_VERSION_PATCH) > VMAF_VERSION_INT_VER(2, 0, 0) + #define CONFIG_LIBVMAF_METADATA_ENABLED 1 #else #define CONFIG_LIBVMAF_METADATA_ENABLED 0 From 102edf979884045b1ca3f25195602fbcf782bade Mon Sep 17 00:00:00 2001 From: Yigithan Yigit Date: Mon, 26 Aug 2024 20:38:56 +0300 Subject: [PATCH 20/20] More bugfix --- libavfilter/vf_libvmaf.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libavfilter/vf_libvmaf.c b/libavfilter/vf_libvmaf.c index 4c5dedc5df5fe..69c1dfdf6bb43 100644 --- a/libavfilter/vf_libvmaf.c +++ b/libavfilter/vf_libvmaf.c @@ -799,16 +799,15 @@ static int activate(AVFilterContext *ctx) if (ff_outlink_get_status(ctx->outputs[0])) s->outlink_eof = 1; - for (unsigned i = 0; i < ctx->nb_inputs; i++) { - if (ff_inlink_acknowledge_status(ctx->inputs[i], &status, &pts)){ - if (!s->flushed) { - ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); - if (ret) - av_log(ctx, AV_LOG_ERROR, - "problem flushing libvmaf context.\n"); - else - s->flushed = 1; - } + if (ff_inlink_acknowledge_status(ctx->inputs[0], &status, &pts) && + ff_inlink_acknowledge_status(ctx->inputs[1], &status, &pts)) { + if (!s->flushed) { + ret = vmaf_read_pictures(s->vmaf, NULL, NULL, 0); + if (ret) + av_log(ctx, AV_LOG_ERROR, + "problem flushing libvmaf context.\n"); + else + s->flushed = 1; } } #endif