ISSUE-111 - More gracefully handle protobuf serialization (#113)

* Fix missing dependency

* [ISSUE-111] properly register protobuf serializer, add ability for UI to render JSON

* minor cleanup

Author: Crim

kafka-webview-ui/pom.xml

@@ -171,6 +171,12 @@
             <artifactId>guava</artifactId>
             <version>27.0.1-jre</version>
         </dependency>
+        <!-- Explicitly include updated protocol buffer version -->
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.6.1</version>
+        </dependency>
 
         <!-- For tests -->
         <dependency>
@@ -217,14 +223,6 @@
             <scope>test</scope>
         </dependency>
 
-        <!-- Test case for validating protocol buffer support in Jackson -->
-        <dependency>
-            <groupId>com.google.protobuf</groupId>
-            <artifactId>protobuf-java</artifactId>
-            <version>3.6.1</version>
-            <scope>test</scope>
-        </dependency>
-
         <!-- Embedded LDAP server for tests -->
         <dependency>
             <groupId>org.zapodot</groupId>

kafka-webview-ui/src/main/frontend/gulp-tasks/build-dist.js

@@ -40,6 +40,7 @@ var vendorsJS = [
   'node_modules/pace-progress/pace.min.js',
   'node_modules/popper.js/dist/umd/popper.min.js',
   'node_modules/popper.js/dist/umd/popper.min.js.map',
+  'node_modules/renderjson/renderjson.js',
   'node_modules/select2/dist/js/select2.min.js',
   'node_modules/toastr/toastr.js',
   'node_modules/handlebars/dist/handlebars.js',

kafka-webview-ui/src/main/frontend/package-lock.json

@@ -14,6 +14,7 @@
     "multiselect-two-sides": "^2.4.0",
     "pace-progress": "^1.0.2",
     "popper.js": "^1.12.5",
+    "renderjson": "^1.4.0",
     "simple-line-icons": "^2.4.1",
     "sockjs-client": "^1.3.0",
     "stompjs": "^2.3.3"

kafka-webview-ui/src/main/frontend/package.json

@@ -24,6 +24,7 @@
 
 package org.sourcelab.kafka.webview.ui.configuration;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
 import org.apache.kafka.common.serialization.Deserializer;
 import org.sourcelab.kafka.webview.ui.manager.encryption.SecretManager;
@@ -36,6 +37,7 @@
 import org.sourcelab.kafka.webview.ui.manager.plugin.UploadManager;
 import org.sourcelab.kafka.webview.ui.manager.sasl.SaslUtility;
 import org.sourcelab.kafka.webview.ui.plugin.filter.RecordFilter;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer;
 import org.springframework.context.annotation.Bean;
 import org.springframework.stereotype.Component;
@@ -122,12 +124,17 @@ public KafkaOperationsFactory getKafkaOperationsFactory(final AppProperties appP
      * @return Jackson2ObjectMapperBuilderCustomizer instance.
      */
     @Bean
-    public Jackson2ObjectMapperBuilderCustomizer addCustomBigDecimalDeserialization() {
+    public Jackson2ObjectMapperBuilderCustomizer registerJacksonProtobufModule() {
         return jacksonObjectMapperBuilder -> {
-            // Register custom protocol buffer serializer as protocol buffers is a common serialization format.
+            // Register custom protocol buffer serializer.
             jacksonObjectMapperBuilder.modulesToInstall(new ProtobufModule());
         };
     }
+
+    @Autowired(required = true)
+    public void configureJackson(ObjectMapper jackson2ObjectMapper) {
+        jackson2ObjectMapper.registerModule(new ProtobufModule());
+    }
 
     /**
      * For creating instances of AdminClient.

kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/PluginConfig.java

@@ -24,11 +24,18 @@
 
 package org.sourcelab.kafka.webview.ui.configuration;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.hubspot.jackson.datatype.protobuf.ProtobufModule;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.http.converter.HttpMessageConverter;
+import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder;
+import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
 import org.springframework.web.servlet.config.annotation.EnableWebMvc;
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurerAdapter;
 
+import java.util.List;
+
 /**
  * Application configuration for Web resources.
  */
@@ -57,4 +64,20 @@ public void addResourceHandlers(ResourceHandlerRegistry registry) {
             .addResourceHandler("/img/**")
             .addResourceLocations("classpath:/static/img/");
     }
+
+    /**
+     * Ensure that ProtobufModule is registered so protobufs can easily be serialized.
+     * @param converters list of converters registered.
+     */
+    @Override
+    public void configureMessageConverters(List<HttpMessageConverter<?>> converters) {
+        converters.clear();
+
+        final ObjectMapper mapper = Jackson2ObjectMapperBuilder
+            .json()
+            .modulesToInstall(new ProtobufModule())
+            .build();
+        converters.add(new MappingJackson2HttpMessageConverter(mapper));
+    }
+
 }

kafka-webview-ui/src/main/java/org/sourcelab/kafka/webview/ui/configuration/WebConfig.java

@@ -53,6 +53,9 @@
     <script src="/vendors/js/jquery-ui.min.js"></script>
     <script src="/vendors/js/jquery-ui-timepicker-addon.min.js"></script>
 
+    <!-- JSON Rendering -->
+    <script src="/vendors/js/renderjson.min.js"></script>
+
     <!-- Websocket -->
     <script src="/vendors/js/sockjs.min.js"></script>
     <script src="/vendors/js/stomp.min.js"></script>

kafka-webview-ui/src/main/resources/templates/layout.html

@@ -109,6 +109,10 @@ <h4 class="alert-heading">Stream Disconnected</h4>
             // End point to consume a view
             consumerBaseUrl: "/user/topic/view",
 
+            // JSON Render settings.
+            useJsonRender: true,
+            jsonRenderToLevel: 1000,
+
             getConsumerUrl : function() {
                 return SocketDetails.consumerBaseUrl + '/' + SocketDetails.viewId + '/' + SocketDetails.userId;
             },
@@ -245,12 +249,44 @@ <h4 class="alert-heading">Stream Disconnected</h4>
 
             // Append results to browser
             appendResult: function(result) {
+                // Determine how to render key
+                var keyIsJson = false;
+                var key = result.key;
+
+                // Determine if result.value is a string, or JSON object
+                if (result.key instanceof Object && !(result.key instanceof String)) {
+                    // Use fancy json rendering.
+                    if (SocketDetails.useJsonRender) {
+                        keyIsJson = true;
+                    } else {
+                        // Convert to String
+                        key = JSON.stringify(result.key);
+                    }
+                }
+
+                // Determine how to render value
+                var valueIsJson = false;
+                var value = result.value;
+
+                // Determine if result.value is a string, or JSON object
+                if (result.value instanceof Object && !(result.value instanceof String)) {
+                    // Use fancy json rendering.
+                    if (SocketDetails.useJsonRender) {
+                        valueIsJson = true;
+                    } else {
+                        // Convert to String
+                        value = JSON.stringify(result.value);
+                    }
+                }
+
                 // Generate html from template
                 var properties = {
                     partition: result.partition,
                     offset: result.offset,
-                    key: result.key,
-                    value: result.value,
+                    key: key,
+                    keyIsJson: keyIsJson,
+                    value: value,
+                    valueIsJson: valueIsJson,
                     timestamp: result.timestamp,
                     date: DateTools.displayTimestamp(result.timestamp),
                     timezone: DateTools.localTimezone,
@@ -261,6 +297,14 @@ <h4 class="alert-heading">Stream Disconnected</h4>
                 // Prepend it to the top of our table
                 SocketDetails.resultsTable.prepend(resultHtml);
 
+                // If json, generate formatted json
+                if (keyIsJson) {
+                    jQuery("td#key-" + result.partition + "-" + result.offset).html(renderjson(result.key));
+                }
+                if (valueIsJson) {
+                    jQuery("td#value-" + result.partition + "-" + result.offset).html(renderjson(result.value));
+                }
+
                 // Remove rows > max results from the end
                 var resultRows = SocketDetails.resultsTable.children();
                 if (resultRows.length > SocketDetails.maxResults) {
@@ -294,6 +338,8 @@ <h4 class="alert-heading">Stream Disconnected</h4>
              * for the selected value.
              */
             jQuery('#startPositionSelector').change(function(event) {
+                renderjson.set_show_to_level(SocketDetails.jsonRenderToLevel);
+
                 var selected = jQuery(this).val();
 
                 // Hide all options
@@ -320,8 +366,12 @@ <h4 class="alert-heading">Stream Disconnected</h4>
                 <span class="timestamp" title="{{date}}">{{timestamp}}</span>
                 {{/if}}
             </td>
-            <td><pre>{{key}}</pre></td>
-            <td><pre>{{value}}</pre></td>
+            <td id="key-{{partition}}-{{offset}}">
+                {{#unless keyIsJson }}<pre>{{key}}</pre>{{/unless}}
+            </td>
+            <td id="value-{{partition}}-{{offset}}">
+                {{#unless valueIsJson }}<pre>{{value}}</pre>{{/unless}}
+            </td>
         </tr>
     </script>
 </section>

kafka-webview-ui/src/main/resources/templates/stream/index.html

@@ -23,10 +23,16 @@
                 viewId: '[[${view.id}]]',
                 formatDates: true,
                 resultsPerPartition: '[[${view.resultsPerPartition}]]',
+
+                // JSON Render settings.
+                useJsonRender: true,
+                jsonRenderToLevel: 1000
             };
 
             // On load, fire off ajax request to load results.
             jQuery(document).ready(function() {
+                renderjson.set_show_to_level(ConsumerInfo.jsonRenderToLevel);
+
                 // Fire off initial consume request
                 executeConsume('next');
 
@@ -160,12 +166,44 @@
 
                 // Loop over results
                 jQuery.each(results.results, function (index, result) {
+                    // Determine how to render key
+                    var keyIsJson = false;
+                    var key = result.key;
+
+                    // Determine if result.value is a string, or JSON object
+                    if (result.key instanceof Object && !(result.key instanceof String)) {
+                        // Use fancy json rendering.
+                        if (ConsumerInfo.useJsonRender) {
+                            keyIsJson = true;
+                        } else {
+                            // Convert to String
+                            key = JSON.stringify(result.key);
+                        }
+                    }
+
+                    // Determine how to render value
+                    var valueIsJson = false;
+                    var value = result.value;
+
+                    // Determine if result.value is a string, or JSON object
+                    if (result.value instanceof Object && !(result.value instanceof String)) {
+                        // Use fancy json rendering.
+                        if (ConsumerInfo.useJsonRender) {
+                            valueIsJson = true;
+                        } else {
+                            // Convert to String
+                            value = JSON.stringify(result.value);
+                        }
+                    }
+
                     // Generate html from template
                     var properties = {
                         partition: result.partition,
                         offset: result.offset,
-                        key: result.key,
-                        value: result.value,
+                        key: key,
+                        keyIsJson: keyIsJson,
+                        value: value,
+                        valueIsJson: valueIsJson,
                         timestamp: result.timestamp,
                         date: DateTools.displayTimestamp(result.timestamp),
                         timezone: DateTools.localTimezone,
@@ -175,6 +213,14 @@
 
                     // Append it to our table
                     jQuery(table).append(resultHtml);
+
+                    // If json, generate formatted json
+                    if (keyIsJson) {
+                        jQuery("td#key-" + result.partition + "-" + result.offset).html(renderjson(result.key));
+                    }
+                    if (valueIsJson) {
+                        jQuery("td#value-" + result.partition + "-" + result.offset).html(renderjson(result.value));
+                    }
                 });
 
                 // Hide loader
@@ -393,8 +439,12 @@ <h4 class="alert-heading">No Results Found</h4>
                     <span class="timestamp" title="{{date}}">{{timestamp}}</span>
                     {{/if}}
                 </td>
-                <td><pre>{{key}}</pre></td>
-                <td><pre>{{value}}</pre></td>
+                <td id="key-{{partition}}-{{offset}}">
+                    {{#unless keyIsJson }}<pre>{{key}}</pre>{{/unless}}
+                </td>
+                <td id="value-{{partition}}-{{offset}}">
+                    {{#unless isJson }}<pre>{{value}}</pre>{{/unless}}
+                </td>
             </tr>
         </script>
     </div>