feat: add JSON output format (#20)

Author: eoinsha

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.9)
 set(CMAKE_CXX_STANDARD 17)
+
 project(duck-query-lambda LANGUAGES CXX)
 
 find_package(aws-lambda-runtime)

Dockerfile

@@ -1,6 +1,7 @@
 ARG TARGETPLATFORM
 ARG TARGETARCH
-FROM public.ecr.aws/sam/build-provided.al2023:latest
+# Ensuring GLIBCXX_3.4.29 as available in Lambda provided.al2023
+FROM public.ecr.aws/sam/build-provided.al2023:1.132
 
 RUN mkdir src
 WORKDIR /src

Makefile

@@ -1,7 +1,7 @@
 PROJECT_NAME = duck-query-lambda
 OUTPUT_DIR = output
 EXTENSIONS = httpfs
-DUCKDB_VERSION = 1.1.3
+DUCKDB_VERSION = 1.2.2
 
 .PHONY: docker_build binary build-bootstrap build-DuckFunction build-DuckFunctionArm
 

README.md

@@ -130,13 +130,14 @@ Here is an example of how to use the Lambda Layer in an AWS Step Function:
 
 ### Invoking the Lambda Function and getting the query results back synchronously
 
-By default, the Lambda function will not return the query results. This is because it's not trivial to convert all results back to JSON in a way that every user expects. If you do want to get the results back synchronously, you can write them to a temporary file in the Lambda and the function will then return the contents of that file, base64 encoded.
+By default, the Lambda function will not return the query results. This is because it's not trivial to convert all results back to JSON in a way that every user expects. If you do want to get the results back synchronously, you can write them to a temporary file in the Lambda and the function will then return the contents of that file. This data is base64 encoded by default, but you can also return the raw JSON from the output file using the `outputFormat` parameter.
 
 Here is an example of how to do this:
 
 ```json
 {
   "query": "COPY (SELECT * FROM 'https://github.com/Teradata/kylo/raw/refs/heads/master/samples/sample-data/parquet/userdata1.parquet' LIMIT 10) TO '/tmp/output.json'",
-  "outputFile": "/tmp/output.json"
+  "outputFile": "/tmp/output.json",
+  "outputFormat": "json"
 }
 ```

bootstrap.cpp

@@ -75,12 +75,28 @@ static invocation_response query_handler(invocation_request const &req, Connecti
     // property, it will be base64 encoded and returned as the result. This temporary file is also deleted before
     // the function completes.
     auto outputFile = view.GetString("outputFile");
+    if (view.KeyExists("outputFormat"))
+    {
+      auto outputFormat = view.GetString("outputFormat");
+      if (outputFormat == "json")
+      {
+        // Read file as JSON string
+        ifstream fileStream(outputFile);
+        if (!fileStream)
+        {
+          throw std::runtime_error("Failed to open file");
+        }
+        std::string jsonString((std::istreambuf_iterator<char>(fileStream)), std::istreambuf_iterator<char>());
+        fileStream.close();
+        remove(outputFile.c_str());
+        return invocation_response::success(jsonString, "application/json");
+      }
+    }
+    // Vanilla base64 encoded response
     auto encodedOutput = b64encode(outputFile);
     // Delete the file after encoding
     remove(outputFile.c_str());
-
-    return invocation_response::success(encodedOutput,
-                                        "application/base64");
+    return invocation_response::success(encodedOutput, "application/base64");
   }
 
   return invocation_response::success(result->ToString(), "text/plain");

examples/sam/Makefile

@@ -0,0 +1,3 @@
+build-DuckQueryFunction:
+	echo "This file is intentionally empty. Everything required is provided by the Lambda layer" > $(ARTIFACTS_DIR)/empty
+

template.yaml

@@ -27,7 +27,7 @@ Resources:
       CompatibleArchitectures:
         - arm64
       CompatibleRuntimes:
-        - provided
+        - provided.al2023
       LicenseInfo: Apache-2.0
       RetentionPolicy: Retain