feat: Pretty print JSON for python. Fix missing regular expressions for masking out ephemeral information (#215)

* feat: Pretty print JSON for python. Fix missing regular expressions for masking out ephemeral information

* feat: Fix backticks in test script. Pin serverless to 3.7

* feat: Mask init duration

* fix: remember to actually commit the removed duration

* feat: update test app to use sls v3

Author: astuyve

.github/workflows/build.yml

@@ -98,7 +98,7 @@ jobs:
           pip install .[dev]
 
       - name: Install Serverless Framework
-        run: sudo yarn global add serverless@^2.72.2 --prefix /usr/local
+        run: sudo yarn global add serverless@^3.7.0 --prefix /usr/local
       - name: Install Crossbuild Deps
         run: |
           sudo apt-get update --allow-releaseinfo-change --fix-missing

scripts/run_integration_tests.sh

@@ -175,14 +175,15 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do
         # Replace invocation-specific data like timestamps and IDs with XXXX to normalize logs across executions
         logs=$(
             echo "$raw_logs" |
+                node parse-json.js |
                 # Filter serverless cli errors
                 sed '/Serverless: Recoverable error occurred/d' |
                 # Remove RequestsDependencyWarning from botocore/vendored/requests/__init__.py
                 sed '/RequestsDependencyWarning/d' |
                 # Remove blank lines
                 sed '/^$/d' |
                 # Normalize Lambda runtime REPORT logs
-                sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
+                sed -E 's/(RequestId|TraceId|SegmentId|Duration|init|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g' |
                 # Normalize HTTP headers
                 sed -E "s/(x-datadog-parent-id:|x-datadog-trace-id:|Content-Length:)[0-9]+/\1XXXX/g" |
                 # Remove Account ID
@@ -204,12 +205,29 @@ for handler_name in "${LAMBDA_HANDLERS[@]}"; do
                 sed -E "s/(\"span_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
                 sed -E "s/(\"parent_id\"\: \")[A-Z0-9\.\-]+/\1XXXX/g" |
                 sed -E "s/(\"request_id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"http.source_ip\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"http.user_agent\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"function_trigger.event_source_arn\"\: \")[A-Za-z0-9\/\.\:\-]+/\1XXXX/g" |
                 sed -E "s/(\"duration\"\: )[0-9\.\-]+/\1\"XXXX\"/g" |
                 sed -E "s/(\"start\"\: )[0-9\.\-]+/\1\"XXXX\"/g" |
                 sed -E "s/(\"system\.pid\"\: )[0-9\.\-]+/\1\"XXXX\"/g" |
                 sed -E "s/(\"runtime-id\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/([a-zA-Z0-9]+)(\.execute-api\.[a-z0-9\-]+\.amazonaws\.com)/XXXX\2/g" |
+                sed -E "s/(\"apiid\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"apiname\"\: \")[a-z0-9\.\-]+/\1XXXX/g" |
+                sed -E "s/(\"function_trigger.event_source_arn\"\: \")[a-z0-9\.\-\:]+/\1XXXX/g" |
+                sed -E "s/(\"event_id\"\: \")[a-zA-Z0-9\:\-]+/\1XXXX/g" |
+                sed -E "s/(\"message_id\"\: \")[a-zA-Z0-9\:\-]+/\1XXXX/g" |
+                sed -E "s/(\"request_id\"\:\ \")[a-zA-Z0-9\-\=]+/\1XXXX/g" |
+                sed -E "s/(\"connection_id\"\:\ \")[a-zA-Z0-9\-]+/\1XXXX/g" |
+                sed -E "s/(\"shardId\-)([0-9]+)\:([a-zA-Z0-9]+)[a-zA-Z0-9]/\1XXXX:XXXX/g" |
+                sed -E "s/(\"shardId\-)[0-9a-zA-Z]+/\1XXXX/g" |
                 sed -E "s/(\"datadog_lambda\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
+                sed -E "s/(\"partition_key\"\:\ \")[a-zA-Z0-9\-]+/\1XXXX/g" |
+                sed -E "s/(\"object_etag\"\:\ \")[a-zA-Z0-9\-]+/\1XXXX/g" |
                 sed -E "s/(\"dd_trace\"\: \")([0-9]+\.[0-9]+\.[0-9])/\1X.X.X/g" |
+                # Parse out account ID in ARN
+                sed -E "s/([a-zA-Z0-9]+):([a-zA-Z0-9]+):([a-zA-Z0-9]+):([a-zA-Z0-9\-]+):([a-zA-Z0-9\-\:]+)/\1:\2:\3:\4:XXXX:\4/g" |
                 sed -E "/init complete at epoch/d" |
                 sed -E "/main started at epoch/d"
         )
@@ -240,7 +258,7 @@ set -e
 if [ "$mismatch_found" = true ]; then
     echo "FAILURE: A mismatch between new data and a snapshot was found and printed above."
     echo "If the change is expected, generate new snapshots by running 'UPDATE_SNAPSHOTS=true DD_API_KEY=XXXX ./scripts/run_integration_tests.sh'"
-    echo "Make sure https://httpstat.us/400/ is UP for `http_error` test case"
+    echo "Make sure https://httpstat.us/400/ is UP for 'http_error' test case"
     exit 1
 fi
 

tests/integration/parse-json.js

@@ -0,0 +1,17 @@
+'use strict'
+
+var readline = require('readline');
+var rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout,
+  terminal: false
+});
+
+rl.on('line', function(line){
+    try {
+      const obj = JSON.parse(line)
+      console.log(JSON.stringify(obj, null, 2))
+    } catch (e) {
+      console.log(line)
+    }
+})

tests/integration/serverless.yml

@@ -1,12 +1,6 @@
 # IAM permissions require service name to begin with 'integration-tests'
 service: integration-tests-python
 
-# As of mid-August 2021, the Serverless framework does not support Python 3.9
-# and complains about an invalid configuration when we deploy Python 3.9 functions.
-# This option suppresses the warning.
-# Remove this when the Serverless framework supports Python 3.9.
-configValidationMode: off
-
 provider:
   name: aws
   region: sa-east-1
@@ -17,8 +11,6 @@ provider:
     DD_TRACE_ENABLED: true
     DD_API_KEY: ${env:DD_API_KEY}
     DD_TRACE_MANAGED_SERVICES: true
-    DD_CAPTURE_LAMBDA_PAYLOAD: true
-  lambdaHashingVersion: 20201221
   timeout: 15
   deploymentBucket:
     name: integration-tests-deployment-bucket