GSoC Dev, v1

libvmaf/include/libvmaf/libvmaf.h

@@ -63,20 +63,20 @@ enum VmafPoolingMethod {
  *
  * @param n_threads   How many threads can be used to run
  *                    feature extractors concurrently.
- * 
+ *
  * @param n_subsample Compute scores only every N frames.
  *                    Note that setting an even value for N can lead to
  *                    inaccurate results. For more detail, see
  *                    https://github.com/Netflix/vmaf/issues/1214
- * 
+ *
  * @param cpumask     Restrict permitted CPU instruction sets.
  *                    if cpumask & 1:  disable SSE2 / disable NEON (on arm64)
  *                    if cpumask & 2:  disable SSE3/SSSE3
  *                    if cpumask & 4:  disable SSE4.1
  *                    if cpumask & 8:  disable AVX2
  *                    if cpumask & 16: disable AVX512
  *                    if cpumask & 32: disable AVX512ICL
- * 
+ *
  * @param gpumask     Restrict permitted GPU operations.
  *                    if gpumask: disable CUDA
  */
@@ -347,6 +347,9 @@ int vmaf_close(VmafContext *vmaf);
 int vmaf_write_output(VmafContext *vmaf, const char *output_path,
                       enum VmafOutputFormat fmt);
 
+int vmaf_propagate_metadata(VmafContext *vmaf, void **metadata, const int frame_idx,
+                            void (*on_features_completed)(void **, const char *, char, float));
+
 /**
  * Get libvmaf version.
  */

libvmaf/src/feature/feature_collector.c

@@ -211,6 +211,13 @@ int vmaf_feature_collector_init(VmafFeatureCollector **const feature_collector)
     fc->feature_vector = malloc(sizeof(*(fc->feature_vector)) * fc->capacity);
     if (!fc->feature_vector) goto free_fc;
     memset(fc->feature_vector, 0, sizeof(*(fc->feature_vector)) * fc->capacity);
+
+    // GSOC 2024: Added for keeping track of last accesed feature vector
+    fc->last_accesed_index = malloc(sizeof(int) * fc->capacity);
+    if (!fc->last_accesed_index) goto free_feature_vector;
+    memset(fc->last_accesed_index, 0, sizeof(int) * fc->capacity);
+    // END GSOC 2024
+
     err = aggregate_vector_init(&fc->aggregate_vector);
     if (err) goto free_feature_vector;
     err = pthread_mutex_init(&(fc->lock), NULL);
@@ -273,8 +280,26 @@ int vmaf_feature_collector_append(VmafFeatureCollector *feature_collector,
             }
             memset(fv + feature_collector->capacity, 0, initial_size);
             feature_collector->feature_vector = fv;
+
+            // GSOC 2024: Added for keeping track of last accesed feature vector
+            initial_size = sizeof(feature_collector->last_accesed_index[0]) *
+                feature_vector->capacity;
+            int *last_accessed_index =
+                realloc(feature_collector->last_accesed_index,
+                sizeof(*(feature_collector->last_accesed_index)) *
+                initial_size * 2);
+            if (!last_accessed_index) {
+                err = -ENOMEM;
+                goto unlock;
+            }
+            memset(last_accessed_index + feature_collector->capacity, 0, initial_size);
+            feature_collector->last_accesed_index = last_accessed_index;
+            // END GSOC 2024
+
             feature_collector->capacity *= 2;
         }
+        feature_collector->last_accesed_index[feature_collector->cnt] = 0;
+
         feature_collector->feature_vector[feature_collector->cnt++]
             = feature_vector;
     }

libvmaf/src/feature/feature_collector.h

@@ -48,6 +48,10 @@ typedef struct VmafFeatureCollector {
     unsigned cnt, capacity;
     struct { clock_t begin, end; } timer;
     pthread_mutex_t lock;
+
+    // GSOC 2024: Last Accessed index per feature vector.
+    int *last_accesed_index; //TODO: ATOMIC?
+    // End GSOC 2024
 } VmafFeatureCollector;
 
 int vmaf_feature_collector_init(VmafFeatureCollector **const feature_collector);

libvmaf/src/libvmaf.c

@@ -32,6 +32,9 @@
 #include "cpu.h"
 #include "feature/feature_extractor.h"
 #include "feature/feature_collector.h"
+
+#include "propagate_metadata.h"
+
 #include "fex_ctx_vector.h"
 #include "log.h"
 #include "model.h"
@@ -41,6 +44,8 @@
 #include "thread_pool.h"
 #include "vcs_version.h"
 
+
+
 #ifdef HAVE_CUDA
 #include "libvmaf/libvmaf_cuda.h"
 
@@ -57,6 +62,7 @@ typedef struct VmafContext {
     VmafFeatureExtractorContextPool *fex_ctx_pool;
     VmafThreadPool *thread_pool;
     VmafFrameSyncContext *framesync;
+    VmafPropagateMetadataContext *metadata_ctx;
 #ifdef HAVE_CUDA
     struct {
         struct {
@@ -106,6 +112,8 @@ int vmaf_init(VmafContext **vmaf, VmafConfiguration cfg)
     if (err) goto free_framesync;
     err = feature_extractor_vector_init(&(v->registered_feature_extractors));
     if (err) goto free_feature_collector;
+    err = vmaf_propagate_metadata_context_init(&(v->metadata_ctx), v->feature_collector);
+    if (err) goto free_metadata_ctx;
 
     if (v->cfg.n_threads > 0) {
         err = vmaf_thread_pool_create(&v->thread_pool, v->cfg.n_threads);
@@ -116,6 +124,8 @@ int vmaf_init(VmafContext **vmaf, VmafConfiguration cfg)
 
     return 0;
 
+free_metadata_ctx:
+    vmaf_propagate_metadata_context_destroy(v->metadata_ctx);
 free_thread_pool:
     vmaf_thread_pool_destroy(v->thread_pool);
 free_feature_extractor_vector:
@@ -907,6 +917,75 @@ int vmaf_score_pooled_model_collection(VmafContext *vmaf,
     return err;
 }
 
+int vmaf_propagate_metadata(VmafContext *vmaf, void **metadata, const int frame_idx,
+                            void (*on_features_completed)(void **, const char *, char, float))
+{
+    if (!vmaf) return -EINVAL;
+    if (!vmaf->feature_collector) return -EINVAL;
+
+    int err, flag;
+    err = flag = 0;
+    VmafPropagateMetadataContext *ctx = vmaf->metadata_ctx;
+
+    if (vmaf->feature_collector->cnt == 0) {
+        printf("No feature extractor registered\n");
+        err = vmaf_frame_queue_push(ctx, frame_idx);
+        return err;
+    }
+
+    // TODO: This is a temporary solution, it should be changed
+    // Basically it is waits for all the feature extractors to be initialized
+    //
+    // Q: Why does this?
+    // A: This is a implementation choice, if we don't wait this and our queue has frames inside of it, it is going to propagate
+    // metadata with ready feature extractors and pop the frame. For example; we have normally 11 feature extractors and at the starting
+    // most of time 2 or 3 of them are ready, so if we continue, it going to propagate those scores and pop the frame which doesnt wait the
+    // remaining ones.
+    for (unsigned f = 0; f < vmaf->feature_collector->cnt; f++) {
+        unsigned i;
+        flag = 0;
+        for (i = 0; i < vmaf->registered_feature_extractors.cnt; i++) {
+            VmafFeatureExtractorContext *fex_ctx = vmaf->registered_feature_extractors.fex_ctx[i];
+            if (fex_ctx->fex->flags & VMAF_FEATURE_EXTRACTOR_CUDA) // It might not be necessary
+                continue;
+            if (strstr(vmaf->feature_collector->feature_vector[f]->name, fex_ctx->fex->name) != NULL) {
+                break;
+            } else {
+                flag++;
+            }
+        }
+        // vmaf->feature_collector->cnt != 11 is hardcoded for now, it supposed to be decided on the fly
+        // they are fixed but it might change depends on the mode
+        if (flag == vmaf->registered_feature_extractors.cnt || vmaf->feature_collector->cnt != 11) {
+            err |= vmaf_frame_queue_push(ctx, frame_idx);
+            printf("Feature extractor not ready\n");
+            return -EBUSY; // Might use a different error code for this
+        }
+    }
+
+    err |= vmaf_frame_queue_push(ctx, frame_idx);
+    if (err) {
+        printf("Error in pushing frame to queue\n");
+        return err;
+    }
+    while (ctx->frame_queue->head != NULL) {
+        VmafFrame frame = vmaf_frame_queue_head(ctx);
+        err |= vmaf_feature_collector_propagate_metadata(ctx, frame.frame_idx, metadata,
+                                                on_features_completed);
+        // TODO: Handle That '-12' arbitrary error code
+        if (err && err != -12) {
+            printf("Error in propagating metadata\n");
+            return err;
+        }
+        if (err != -12) {
+            vmaf_frame_queue_pop(ctx);
+            continue;
+        }
+        break;
+    }
+    return err;
+}
+
 const char *vmaf_version(void)
 {
     return VMAF_VERSION;

libvmaf/src/meson.build

@@ -34,7 +34,7 @@ is_cuda_enabled = get_option('enable_cuda') == true
 is_avx512_enabled = get_option('enable_avx512') == true
 is_nvtx_enabled = get_option('enable_nvtx') == true
 
-if is_nvtx_enabled    
+if is_nvtx_enabled
     cdata.set10('HAVE_NVTX', is_nvtx_enabled)
 endif
 
@@ -462,6 +462,7 @@ libvmaf_sources = [
     src_dir + 'pdjson.c',
     src_dir + 'log.c',
     src_dir + 'framesync.c',
+    src_dir + 'propagate_metadata.c'
 ]
 
 if is_cuda_enabled

libvmaf/src/propagate_metadata.c

@@ -0,0 +1,167 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+
+#include "propagate_metadata.h"
+#include "dict.h"
+#include "errno.h"
+
+
+int vmaf_propagate_metadata_context_init(VmafPropagateMetadataContext **ctx, VmafFeatureCollector *fc)
+{
+    if (!ctx) return -EINVAL;
+
+    *ctx = malloc(sizeof(VmafPropagateMetadataContext));
+    if (!*ctx) return -ENOMEM;
+
+    (*ctx)->frame_queue = malloc(sizeof(VmafFrameQueue));
+    if (!(*ctx)->frame_queue) return -ENOMEM;
+
+    (*ctx)->fc = fc;
+    vmaf_frame_queue_init(&((*ctx)->frame_queue));
+
+    pthread_mutex_init(&((*ctx)->lock), NULL);
+
+    return 0;
+}
+
+int vmaf_propagate_metadata_context_destroy(VmafPropagateMetadataContext *ctx)
+{
+    if (!ctx) return -EINVAL;
+
+    while (ctx->frame_queue->head) {
+        VmafFrame frame = vmaf_frame_queue_pop(ctx);
+    }
+    free(ctx->frame_queue);
+    free(ctx);
+    pthread_mutex_destroy(&(ctx->lock));
+
+    return 0;
+}
+
+int vmaf_frame_queue_init(VmafFrameQueue **frame_queue)
+{
+    if (!frame_queue) return -EINVAL;
+
+    *frame_queue = malloc(sizeof(VmafFrameQueue));
+    if (!*frame_queue) return -ENOMEM;
+
+    (*frame_queue)->head = NULL;
+    (*frame_queue)->tail = NULL;
+
+    return 0;
+}
+
+int vmaf_frame_queue_push(VmafPropagateMetadataContext *ctx, const int frame_idx)
+{
+    if (!ctx) return -EINVAL;
+
+    pthread_mutex_lock(&(ctx->lock));
+
+    VmafFrameQueue *frame_queue = ctx->frame_queue;
+    VmafFrame *new_frame = malloc(sizeof(VmafFrame));
+    if (!new_frame) {
+        pthread_mutex_unlock(&(ctx->lock));
+        return -ENOMEM;
+    }
+
+    new_frame->frame_idx = frame_idx;
+    new_frame->next = NULL;
+
+    if (!frame_queue->head) {
+        frame_queue->head = new_frame;
+        frame_queue->tail = new_frame;
+    } else {
+        frame_queue->tail->next = new_frame;
+        frame_queue->tail = new_frame;
+    }
+
+    pthread_mutex_unlock(&(ctx->lock));
+    return 0;
+}
+
+VmafFrame vmaf_frame_queue_pop(VmafPropagateMetadataContext *ctx)
+{
+    if (!ctx) {
+        VmafFrame empty_frame = { .frame_idx = -1 };
+        return empty_frame;
+    }
+
+    pthread_mutex_lock(&(ctx->lock));
+
+    VmafFrameQueue *frame_queue = ctx->frame_queue;
+    VmafFrame *next_frame = frame_queue->head;
+
+    if (!next_frame) {
+        VmafFrame empty_frame = { .frame_idx = -1 };
+        pthread_mutex_unlock(&(ctx->lock));
+        return empty_frame;
+    }
+
+    VmafFrame popped_frame = *next_frame;
+
+    if (frame_queue->head == frame_queue->tail) {
+        frame_queue->head = NULL;
+        frame_queue->tail = NULL;
+    } else {
+        frame_queue->head = next_frame->next;
+    }
+
+    free(next_frame);
+
+    pthread_mutex_unlock(&(ctx->lock));
+
+    return popped_frame;
+}
+
+VmafFrame vmaf_frame_queue_head(VmafPropagateMetadataContext *ctx)
+{
+    if (!ctx) {
+        VmafFrame empty_frame = { .frame_idx = -1 };
+        return empty_frame;
+    };
+
+    pthread_mutex_lock(&(ctx->lock));
+
+    VmafFrameQueue *frame_queue = ctx->frame_queue;
+    VmafFrame *next_frame = frame_queue->head;
+
+    if (!next_frame) {
+        VmafFrame empty_frame = { .frame_idx = -1 };
+        pthread_mutex_unlock(&(ctx->lock));
+        return empty_frame;
+    }
+
+    VmafFrame popped_frame = *next_frame;
+
+    pthread_mutex_unlock(&(ctx->lock));
+
+    return popped_frame;
+}
+
+int vmaf_feature_collector_propagate_metadata(VmafPropagateMetadataContext *ctx, const int frame_idx, void **metadata, void (*on_features_completed)(void **, const char *, const char *))
+{
+    if (!ctx) return -EINVAL;
+    if (!ctx->fc)  return -EINVAL;
+    if (frame_idx < 0) return -EINVAL;
+
+    double score = 0.0f;
+    int err = 0;
+    unsigned i;
+    VmafFeatureCollector *fc = ctx->fc;
+
+    for (i = 0; i < fc->cnt; i++) {
+        err = vmaf_feature_collector_get_score(fc, fc->feature_vector[i]->name, &score, frame_idx);
+        if (err) {
+            return -12; // Arbitrary error code that I defined
+        }
+        //printf("Feature Index: %d Feature name: %s, score: %f\n", frame_idx, fc->feature_vector[i]->name, score);
+        char value[128];
+        snprintf(value, sizeof(value), "%.6f", score);
+        on_features_completed(metadata, fc->feature_vector[i]->name, value);
+    }
+
+    return 0;
+}