From 47fd73e2542b5c6a0ea22a9633efe08ebd01532a Mon Sep 17 00:00:00 2001 From: rameshm <94033485+ramesh-maddegoda@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:20:55 -0700 Subject: [PATCH 1/3] ADD: streaming-json report style for line-delimited product validation - Implement StreamingJsonReport class to output validation results as line-delimited JSON (NDJSON) to stdout or a file. - Add error, fatal, warning, and info metrics tracking per product line to streamline AWS CloudWatch log filtering. - Update ValidateLauncher to support the new 'streaming-json' style flag option and route report setup accordingly. - Fix WSL2 file permission tracking clutter by configuring repository core.fileMode to false. Related: https://github.com/NASA-PDS/validate/issues/1352 --- .../nasa/pds/validate/ValidateLauncher.java | 27 +++-- .../validate/report/StreamingJsonReport.java | 111 ++++++++++++++++++ 2 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java diff --git a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java index 00b802f72..96a09df51 100644 --- a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java +++ b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java @@ -1,4 +1,4 @@ -// Copyright © 2019, California Institute of Technology ("Caltech"). +// Copyright © 2026, California Institute of Technology ("Caltech"). // U.S. Government sponsorship acknowledged. // // All rights reserved. @@ -121,6 +121,7 @@ import gov.nasa.pds.validate.report.JSONReport; import gov.nasa.pds.validate.report.Report; import gov.nasa.pds.validate.report.XmlReport; +import gov.nasa.pds.validate.report.StreamingJsonReport; import gov.nasa.pds.validate.util.ToolInfo; import gov.nasa.pds.validate.util.Utility; @@ -1022,15 +1023,18 @@ public void setCatalogs(List catalogs) { /** * Set the output style for the report. * - * @param style 'sum' for a summary report, 'min' for a minimal report, and 'full' for a full - * report - * @throws Exception + * @param style 'full' for a full report, 'json' for a buffered JSON report, + * 'xml' for an XML report, or 'streaming-json' for a line-delimited + * JSON report (one object per product). + * @throws Exception If an invalid style is provided. */ public void setReportStyle(String style) throws Exception { - if (!style.equalsIgnoreCase("full") && !style.equalsIgnoreCase("json") - && !style.equalsIgnoreCase("xml")) { + if (!style.equalsIgnoreCase("full") && + !style.equalsIgnoreCase("json") && + !style.equalsIgnoreCase("xml") && + !style.equalsIgnoreCase("streaming-json")) { throw new Exception("Invalid value entered for 's' flag. Value can only " - + "be either 'full', 'json' or 'xml'"); + + "be 'full', 'json', 'xml', or 'streaming-json'"); } this.reportStyle = style; } @@ -1316,13 +1320,16 @@ public void displayVersion() throws IOException { * @throws IOException If an error occurred while setting up the report. */ public void setupReport(String cliArgs[]) throws IOException { - if (this.reportStyle.equals("full")) { + if (this.reportStyle.equalsIgnoreCase("full")) { this.report = new FullReport(); - } else if (this.reportStyle.equals("json")) { + } else if (this.reportStyle.equalsIgnoreCase("json")) { this.report = new JSONReport(); - } else if (this.reportStyle.equals("xml")) { + } else if (this.reportStyle.equalsIgnoreCase("xml")) { this.report = new XmlReport(); + } else if (this.reportStyle.equalsIgnoreCase("streaming-json")) { // ADD THIS + this.report = new StreamingJsonReport(); } + report.setLevel(severity); if (reportFile != null) { report.setWriter(new PrintWriter(reportFile)); diff --git a/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java b/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java new file mode 100644 index 000000000..a0b67abdd --- /dev/null +++ b/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java @@ -0,0 +1,111 @@ +package gov.nasa.pds.validate.report; + +import java.net.URI; +import java.util.List; +import java.util.ArrayList; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; + +import gov.nasa.pds.tools.validate.ValidationProblem; +import gov.nasa.pds.tools.label.ExceptionType; +import gov.nasa.pds.validate.status.Status; + +public class StreamingJsonReport extends Report { + private static final Gson gson = new Gson(); + private JsonObject currentProduct; + private JsonArray currentMessages; + + @Override + protected void begin(Block block) { + if (block == Block.LABEL) { + this.currentProduct = new JsonObject(); + this.currentMessages = new JsonArray(); + } + } + + @Override + protected void append(Status status, String lidvid, String target) { + this.currentProduct.addProperty("label", target); + this.currentProduct.addProperty("lidvid", lidvid); + this.currentProduct.addProperty("status", status.getName()); + } + + @Override + protected void append(ValidationProblem problem) { + JsonObject msg = new JsonObject(); + msg.addProperty("severity", problem.getProblem().getSeverity().getName()); + msg.addProperty("type", problem.getProblem().getType().getKey()); + msg.addProperty("message", problem.getMessage()); + + if (problem.getLineNumber() != -1) { + msg.addProperty("line", problem.getLineNumber()); + if (problem.getColumnNumber() != -1) { + msg.addProperty("column", problem.getColumnNumber()); + } + } + this.currentMessages.add(msg); + } + + @Override + protected void end(Block block) { + if (block == Block.LABEL) { + // Calculate error/warning counts from the collected messages + int fatals = 0, errors = 0, warnings = 0, infos = 0; + + for (JsonElement element : this.currentMessages) { + String severity = element.getAsJsonObject().get("severity").getAsString(); + if ("FATAL".equalsIgnoreCase(severity)) fatals++; + else if ("ERROR".equalsIgnoreCase(severity)) errors++; + else if ("WARNING".equalsIgnoreCase(severity)) warnings++; + else if ("INFO".equalsIgnoreCase(severity)) infos++; + } + + this.currentProduct.addProperty("fatal", fatals); + this.currentProduct.addProperty("error", errors); + this.currentProduct.addProperty("warning", warnings); + this.currentProduct.addProperty("info", infos); + + this.currentProduct.add("messages", this.currentMessages); + + // Write to the configured output stream (file or console) + if (getWriter() != null) { + getWriter().println(gson.toJson(this.currentProduct)); + getWriter().flush(); + } else { + System.out.println(gson.toJson(this.currentProduct)); + } + + // Clear memory + this.currentProduct = null; + this.currentMessages = null; + } + } + + // --- Implement remaining abstract methods as empty --- + + @Override protected void append(String title) {} + @Override protected void appendConfig(String label, String message, String value) {} + @Override protected void appendParam(String label, String message, String value) {} + @Override protected void summarizeAddMessage(String msg, long count) {} + @Override protected void summarizeDepWarn(String msg) {} + @Override protected void summarizeRefs(int failed, int passed, int skipped, int total) {} + @Override protected void summarizeTotals(int errors, int total, int warnings) {} + + @Override + protected void summarizeProds(int failed, int passed, int skipped, int total) { + JsonObject summary = new JsonObject(); + summary.addProperty("recordType", "summary"); + summary.addProperty("passed", passed); + summary.addProperty("failed", failed); + summary.addProperty("total", total); + + if (getWriter() != null) { + getWriter().println(gson.toJson(summary)); + getWriter().flush(); + } else { + System.out.println(gson.toJson(summary)); + } + } +} \ No newline at end of file From 13c7e75d8c5d5ae7865607e53567019422f3f3f0 Mon Sep 17 00:00:00 2001 From: rameshm <94033485+ramesh-maddegoda@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:32:03 -0700 Subject: [PATCH 2/3] UPDATE: code to remove unwanted comment Related: https://github.com/NASA-PDS/validate/issues/1352 --- src/main/java/gov/nasa/pds/validate/ValidateLauncher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java index 96a09df51..285cf84eb 100644 --- a/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java +++ b/src/main/java/gov/nasa/pds/validate/ValidateLauncher.java @@ -1326,7 +1326,7 @@ public void setupReport(String cliArgs[]) throws IOException { this.report = new JSONReport(); } else if (this.reportStyle.equalsIgnoreCase("xml")) { this.report = new XmlReport(); - } else if (this.reportStyle.equalsIgnoreCase("streaming-json")) { // ADD THIS + } else if (this.reportStyle.equalsIgnoreCase("streaming-json")) { this.report = new StreamingJsonReport(); } From bf186ef634851d0632b83b511f347916d6bea0b8 Mon Sep 17 00:00:00 2001 From: rameshm <94033485+ramesh-maddegoda@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:35:56 -0700 Subject: [PATCH 3/3] UPDATE to incorporate sonar findings. Related: https://github.com/NASA-PDS/validate/issues/1352 --- .../validate/report/StreamingJsonReport.java | 61 +++++++++++++------ 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java b/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java index a0b67abdd..5de5bc4e4 100644 --- a/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java +++ b/src/main/java/gov/nasa/pds/validate/report/StreamingJsonReport.java @@ -1,18 +1,16 @@ package gov.nasa.pds.validate.report; -import java.net.URI; -import java.util.List; -import java.util.ArrayList; +import java.util.logging.Logger; import com.google.gson.Gson; import com.google.gson.JsonObject; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import gov.nasa.pds.tools.validate.ValidationProblem; -import gov.nasa.pds.tools.label.ExceptionType; import gov.nasa.pds.validate.status.Status; public class StreamingJsonReport extends Report { + private static final Logger LOG = Logger.getLogger(StreamingJsonReport.class.getName()); private static final Gson gson = new Gson(); private JsonObject currentProduct; private JsonArray currentMessages; @@ -51,8 +49,10 @@ protected void append(ValidationProblem problem) { @Override protected void end(Block block) { if (block == Block.LABEL) { - // Calculate error/warning counts from the collected messages - int fatals = 0, errors = 0, warnings = 0, infos = 0; + int fatals = 0; + int errors = 0; + int warnings = 0; + int infos = 0; for (JsonElement element : this.currentMessages) { String severity = element.getAsJsonObject().get("severity").getAsString(); @@ -69,29 +69,54 @@ protected void end(Block block) { this.currentProduct.add("messages", this.currentMessages); - // Write to the configured output stream (file or console) if (getWriter() != null) { getWriter().println(gson.toJson(this.currentProduct)); getWriter().flush(); } else { - System.out.println(gson.toJson(this.currentProduct)); + LOG.info(gson.toJson(this.currentProduct)); } - // Clear memory this.currentProduct = null; this.currentMessages = null; } } - // --- Implement remaining abstract methods as empty --- + // --- Implement abstract overrides with explicit comments for SonarQube --- - @Override protected void append(String title) {} - @Override protected void appendConfig(String label, String message, String value) {} - @Override protected void appendParam(String label, String message, String value) {} - @Override protected void summarizeAddMessage(String msg, long count) {} - @Override protected void summarizeDepWarn(String msg) {} - @Override protected void summarizeRefs(int failed, int passed, int skipped, int total) {} - @Override protected void summarizeTotals(int errors, int total, int warnings) {} + @Override + protected void append(String title) { + // Intentionally empty: Title blocks are excluded from line-delimited JSON log output + } + + @Override + protected void appendConfig(String label, String message, String value) { + // Intentionally empty: Global tool setup configurations are excluded from streaming records + } + + @Override + protected void appendParam(String label, String message, String value) { + // Intentionally empty: Run parameter notifications are excluded from streaming records + } + + @Override + protected void summarizeAddMessage(String msg, long count) { + // Intentionally empty: Aggregate counts are dropped in favor of line-item logs + } + + @Override + protected void summarizeDepWarn(String msg) { + // Intentionally empty: Deprecation warnings are handled in global stdout rather than streaming targets + } + + @Override + protected void summarizeRefs(int failed, int passed, int skipped, int total) { + // Intentionally empty: Context references are skipped to isolate file validation metrics + } + + @Override + protected void summarizeTotals(int errors, int total, int warnings) { + // Intentionally empty: Replaced by target summary record type handled in summarizeProds + } @Override protected void summarizeProds(int failed, int passed, int skipped, int total) { @@ -105,7 +130,7 @@ protected void summarizeProds(int failed, int passed, int skipped, int total) { getWriter().println(gson.toJson(summary)); getWriter().flush(); } else { - System.out.println(gson.toJson(summary)); + LOG.info(gson.toJson(summary)); } } } \ No newline at end of file