From 29c66cbc7fa070ce25a5fab9fdde058de5b1ec16 Mon Sep 17 00:00:00 2001
From: Aron Helser <aron.helser@kitware.com>
Date: Fri, 5 Jul 2024 11:17:10 -0400
Subject: [PATCH] fix(RawImageDisplayArea): add ack_id echo and user_data event
 when frame received

When a frame is sent by the server to the RawImageDisplayArea, allow
the inclusion of `ack_id` in the meta information, which is sent
as message back to the server when the frame is received. Allows the
server to track if the client is keeping up with the frames sent.

Also allow `user_data` in the meta information, which is propagated
to the client as a Vue event.
---
 trame_rca/protocol.py                         | 10 ++++++++
 vue-components/src/components/DisplayArea.js  |  3 ++-
 .../src/components/RawImageDisplayArea.js     | 25 ++++++++++++++++---
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/trame_rca/protocol.py b/trame_rca/protocol.py
index 9192fdc..1ccd1b1 100644
--- a/trame_rca/protocol.py
+++ b/trame_rca/protocol.py
@@ -71,3 +71,13 @@ def on_interaction(self, area_name, origin, event):
             adapter.on_interaction(origin, event)
         else:
             print(f"No area {area_name} available for event")
+
+    @exportRpc("trame.rca.ack_id")
+    def on_ack_id(self, area_name, ack_id):
+        # if ack_id is sent with a frame, it is echoed back to the
+        # adapter when the frame is received.
+        adapter = self._area_adapters.get(area_name, None)
+        if adapter:
+            adapter.on_ack_id(ack_id)
+        else:
+            print(f"No area {area_name} available for event")
diff --git a/vue-components/src/components/DisplayArea.js b/vue-components/src/components/DisplayArea.js
index 55af60e..954e4b5 100644
--- a/vue-components/src/components/DisplayArea.js
+++ b/vue-components/src/components/DisplayArea.js
@@ -29,7 +29,8 @@ export default {
       <image-display-area v-if="display === 'image'" :name="name" :origin="origin" :poolSize="4" />
       <media-source-display-area v-if="display === 'media-source'" :name="name" :origin="origin" />
       <video-decoder-display-area v-if="display === 'video-decoder'" :name="name" :origin="origin" />
-      <raw-image-display-area v-if="display === 'raw-image'" :name="name" :origin="origin" />
+      <raw-image-display-area v-if="display === 'raw-image'" :name="name" :origin="origin"
+        @user_data="(d) => $emit('user_data', d)"/>
     </div>
   `,
 };
diff --git a/vue-components/src/components/RawImageDisplayArea.js b/vue-components/src/components/RawImageDisplayArea.js
index bcd865e..4749894 100644
--- a/vue-components/src/components/RawImageDisplayArea.js
+++ b/vue-components/src/components/RawImageDisplayArea.js
@@ -1,3 +1,5 @@
+const { inject } = window.Vue;
+
 export default {
   props: {
     name: {
@@ -29,18 +31,21 @@ export default {
     },
   },
   mounted() {
+    const trame = inject('trame');
+
     this.wslinkSubscription = null;
     const canvas = this.$el;
     const ctx = canvas.getContext('2d');
     this.onImage = async ([{ name, meta, content }]) => {
       if (this.name === name) {
+        let imageData = null;
         if (meta.type.includes('image/rgb24')) {
           const data = content.buffer
             ? content
             : new Uint8Array(await content.arrayBuffer());
           canvas.width = meta.w;
           canvas.height = meta.h;
-          const imageData = ctx.createImageData(meta.w, meta.h);
+          imageData = ctx.createImageData(meta.w, meta.h);
           const pixels = imageData.data;
           let iRGB = 0;
           let iRGBA = 0;
@@ -50,16 +55,28 @@ export default {
             pixels[iRGBA++] = data[iRGB++];
             pixels[iRGBA++] = 255;
           }
-          ctx.putImageData(imageData, 0, 0);
-          this.hasContent = true;
         } else if (meta.type.includes('image/rgba32')) {
           const data = new Uint8ClampedArray(
             content.buffer ? content : await content.arrayBuffer()
           );
           canvas.width = meta.w;
           canvas.height = meta.h;
-          const imageData = new ImageData(data, meta.w, meta.h);
+          imageData = new ImageData(data, meta.w, meta.h);
+        }
+        if (imageData) {
           ctx.putImageData(imageData, 0, 0);
+          // if the frame sender provided an ack_id, send back to the server
+          // so they know this frame has been processed
+          if (meta['ack_id'] !== undefined) {
+            trame.client
+              .getConnection()
+              .getSession()
+              .call('trame.rca.ack_id', [this.name, meta['ack_id']]);
+          }
+          // if the frame sender provide user_data, emit as a client event
+          if (meta['user_data'] !== undefined) {
+            this.$emit('user_data', meta['user_data']);
+          }
           this.hasContent = true;
         } else {
           this.hasContent = false;