Return HTTP 413 if response exceeds configured MAXIMUM_BYTES_RETURNED

service/configuration.json.sample

@@ -36,5 +36,6 @@
     "COLLECTION": "daily_streams",
     "C_COLLECTION": "c_segments"
   },
-  "CLOSED": false
+  "CLOSED": false,
+  "VERSION": "1.0.3",
 }

service/runtests.js

@@ -691,6 +691,39 @@ var suite = function () {
       })
       .end();
   };
+
+  /*
+  * Tests an response too large to return as configured by MAXIMUM_BYTES_RETURNED > 0
+  */
+  this.testPayloadTooLarge = function(callback) {
+    const options = getOptions(
+      "GET",
+      CONFIG.BASE_URL +
+        "query?network=NL&station=G233&include=sample&start=2024-01-01&end=2024-01-31"
+    );
+
+    const prevMaxBytes = CONFIG.MAXIMUM_BYTES_RETURNED
+    CONFIG.MAXIMUM_BYTES_RETURNED = 10e3
+
+    http
+      .request(options, function (response) {
+        response.on("data", function (data) {
+          const err = compareResponse(data, {
+            message: {
+              code: ERROR.MAXIMUM_PAYLOAD_EXCEEDED.code,
+              msg: ERROR.MAXIMUM_PAYLOAD_EXCEEDED.msg
+            },
+            request: options.path
+          });
+
+          CONFIG.MAXIMUM_BYTES_RETURNED = prevMaxBytes
+
+          callback(err);
+        });
+      })
+      .end();
+
+  };
 };
 
 function compareResponse(a, b) {

service/server.js

@@ -64,7 +64,7 @@ module.exports = function (CONFIG, WFCatalogCallback) {
   var WFCatalogger;
   setupLogger();
 
-  const VERSION = "1.0.2"
+  const VERSION = "1.0.3"
 
   // The service is powered by express
   var WFCatalog = require("express")();
@@ -692,6 +692,7 @@ module.exports = function (CONFIG, WFCatalogCallback) {
     req.WFCatalog.nDocuments = 0;
     req.WFCatalog.nBytes = 0;
     req.WFCatalog.nContinuous = 0;
+    req.WFCatalog.responseChunks = [];
 
     // Define variables for hoisting
     var documentPointer;
@@ -740,10 +741,10 @@ module.exports = function (CONFIG, WFCatalogCallback) {
       // or the document is one trace, write the daily metric document
       // and proceed along the cursor
       if (!req.WFCatalog.options.csegments || doc.cont) {
-        if (writeStream(req, res, documentPointer)) {
+        if (buildResponse(req, res, documentPointer)) {
           return cursor.next(processDailyStream);
         }
-        return endResponse(req, res);
+        return sendErrorPage(req, res, ERROR.MAXIMUM_PAYLOAD_EXCEEDED);
       }
 
       // We are required to collect continuous segments
@@ -792,11 +793,11 @@ module.exports = function (CONFIG, WFCatalogCallback) {
 
       // The cursor has been exhausted; write to stream
       // and proceed with the next daily stream
-      if (writeStream(req, res, documentPointer)) {
+      if (buildResponse(req, res, documentPointer)) {
         return cursor.next(processDailyStream);
       }
 
-      return endResponse(req, res);
+      return sendErrorPage(req, res, ERROR.MAXIMUM_PAYLOAD_EXCEEDED);
     }
   });
 
@@ -833,28 +834,48 @@ module.exports = function (CONFIG, WFCatalogCallback) {
     }
   });
 
-  /* @ function writeStream
-   * Handler for writing to the writeable response stream
-   * counts the nBytes shipped and properly parses the
-   * JSON body
+  /**
+   * @ function writeStream
+   * Handler for writing response stream
+   * 
+   * @param {Object} req: Express Request Object
+   * @param {Object} res: Express Response Object
+   * @param {object} data: Stringified JSON Query result
+   * 
    */
   function writeStream(req, res, data) {
-    var json = JSON.stringify(data);
-    req.WFCatalog.nBytes += Buffer.byteLength(json);
-
     // Delimit a document by a comma or open the JSON
     if (req.WFCatalog.nDocuments === 1) {
       res.write("[");
     } else {
       res.write(",");
     }
 
-    res.write(json);
+    req.WFCatalog.nBytes += 1;
+    res.write(data);
+  }
 
-    // Set to 0 for no maximum
+  /**
+   * @ function buildResponse 
+   * Handler for building the response.
+   * Checks if the response is within set limits
+   * 
+   * @param {Object} req: Express Request Object
+   * @param {Object} res: Express Response Object
+   * @param {object} data: Query result in JSON
+   * 
+   */
+  function buildResponse(req, res, data) {
+    var json = JSON.stringify(data);
+    req.WFCatalog.nBytes += Buffer.byteLength(json);
+
+    // No limit on returnable data, therefore stream the response
     if (CONFIG.MAXIMUM_BYTES_RETURNED === 0) {
+      writeStream(req, res, json);
       return true;
     }
+
+    req.WFCatalog.responseChunks.push(data);
 
     return req.WFCatalog.nBytes < CONFIG.MAXIMUM_BYTES_RETURNED;
   }
@@ -878,10 +899,16 @@ module.exports = function (CONFIG, WFCatalogCallback) {
       return res.status(req.WFCatalog.options.nodata).end();
     }
 
-    // Close JSON and celebrate a succesful request
-    res.write("]");
+    // No limit of returnable data was defined, close the stream and return
+    if (CONFIG.MAXIMUM_BYTES_RETURNED === 0) {
+      res.write("]");
+      req.WFCatalog.nBytes += 1;
+
+      return res.end();
+    }
 
-    return res.status(200).end();
+    // A limit of returnable data was defined
+    return res.status(200).json(req.WFCatalog.responseChunks);
   }
 
   /*