From 63c73dda021b2b6d108faa565ffea9b83883677a Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 12:14:50 +0100 Subject: [PATCH 01/10] Move Agentless cloudformation templates to a dedicated folder --- .../datadog_agentless_api_call.py | 1 + .../datadog_agentless_api_call_test.py | 1 + .../datadog_agentless_delegate_role.yaml | 1 + ...adog_agentless_delegate_role_snapshot.yaml | 1 + .../datadog_agentless_scanning.yaml | 1 + aws_agentless/release.sh | 82 +++++++++++++++++++ aws_agentless/version.txt | 1 + aws_quickstart/release.sh | 20 ----- 8 files changed, 88 insertions(+), 20 deletions(-) rename {aws_quickstart => aws_agentless}/datadog_agentless_api_call.py (99%) rename {aws_quickstart => aws_agentless}/datadog_agentless_api_call_test.py (99%) rename {aws_quickstart => aws_agentless}/datadog_agentless_delegate_role.yaml (99%) rename {aws_quickstart => aws_agentless}/datadog_agentless_delegate_role_snapshot.yaml (99%) rename {aws_quickstart => aws_agentless}/datadog_agentless_scanning.yaml (99%) create mode 100755 aws_agentless/release.sh create mode 100644 aws_agentless/version.txt diff --git a/aws_quickstart/datadog_agentless_api_call.py b/aws_agentless/datadog_agentless_api_call.py similarity index 99% rename from aws_quickstart/datadog_agentless_api_call.py rename to aws_agentless/datadog_agentless_api_call.py index 40356628..09af5ac5 100644 --- a/aws_quickstart/datadog_agentless_api_call.py +++ b/aws_agentless/datadog_agentless_api_call.py @@ -198,3 +198,4 @@ def timeout_handler(_signal, _frame): signal.signal(signal.SIGALRM, timeout_handler) + diff --git a/aws_quickstart/datadog_agentless_api_call_test.py b/aws_agentless/datadog_agentless_api_call_test.py similarity index 99% rename from aws_quickstart/datadog_agentless_api_call_test.py rename to aws_agentless/datadog_agentless_api_call_test.py index 1187b898..85fa6e5c 100644 --- a/aws_quickstart/datadog_agentless_api_call_test.py +++ b/aws_agentless/datadog_agentless_api_call_test.py @@ -246,3 +246,4 @@ def test_other_error_raises_exception(self, mock_urlopen): if __name__ == "__main__": unittest.main() + diff --git a/aws_quickstart/datadog_agentless_delegate_role.yaml b/aws_agentless/datadog_agentless_delegate_role.yaml similarity index 99% rename from aws_quickstart/datadog_agentless_delegate_role.yaml rename to aws_agentless/datadog_agentless_delegate_role.yaml index d7c0c76a..464f8f65 100644 --- a/aws_quickstart/datadog_agentless_delegate_role.yaml +++ b/aws_agentless/datadog_agentless_delegate_role.yaml @@ -360,3 +360,4 @@ Metadata: Parameters: - AgentlessSensitiveDataScanning - AccountId + diff --git a/aws_quickstart/datadog_agentless_delegate_role_snapshot.yaml b/aws_agentless/datadog_agentless_delegate_role_snapshot.yaml similarity index 99% rename from aws_quickstart/datadog_agentless_delegate_role_snapshot.yaml rename to aws_agentless/datadog_agentless_delegate_role_snapshot.yaml index 711f674d..86e8b06f 100644 --- a/aws_quickstart/datadog_agentless_delegate_role_snapshot.yaml +++ b/aws_agentless/datadog_agentless_delegate_role_snapshot.yaml @@ -44,3 +44,4 @@ Resources: 'aws:TagKeys': DatadogAgentlessScanner* StringEquals: 'aws:RequestTag/DatadogAgentlessScanner': 'true' + diff --git a/aws_quickstart/datadog_agentless_scanning.yaml b/aws_agentless/datadog_agentless_scanning.yaml similarity index 99% rename from aws_quickstart/datadog_agentless_scanning.yaml rename to aws_agentless/datadog_agentless_scanning.yaml index c167d2d5..10ba55c5 100644 --- a/aws_quickstart/datadog_agentless_scanning.yaml +++ b/aws_agentless/datadog_agentless_scanning.yaml @@ -1131,3 +1131,4 @@ Metadata: default: "AgentlessLambdaScanning *" AgentlessSensitiveDataScanning: default: "AgentlessSensitiveDataScanning *" + diff --git a/aws_agentless/release.sh b/aws_agentless/release.sh new file mode 100755 index 00000000..6c0fb717 --- /dev/null +++ b/aws_agentless/release.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# Usage: ./release.sh + +set -e + +# Read the S3 bucket +if [ -z "$1" ]; then + echo "Must specify a S3 bucket to publish the template" + exit 1 +else + BUCKET=$1 +fi + +# Read the version +VERSION=$(head -n 1 version.txt) + +# Confirm the bucket for the current release doesn't already exist so we don't overwrite it +set +e +EXIT_CODE=0 +response=$(aws s3api head-object \ + --bucket "${BUCKET}" \ + --key "aws_agentless/${VERSION}/datadog_agentless_scanning.yaml" > /dev/null 2>&1) + +if [[ ${?} -eq 0 ]]; then + echo "S3 bucket path ${BUCKET}/aws_agentless/${VERSION} already exists. Please up the version." + exit 1 +fi +set -e + +# Upload templates to a private bucket -- useful for testing +if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then + PRIVATE_TEMPLATE=true +else + PRIVATE_TEMPLATE=false +fi + +# Confirm to proceed +for i in *.yaml; do + [ -f "$i" ] || break + echo "About to upload $i to s3://${BUCKET}/aws_agentless/${VERSION}/$i" +done +read -p "Continue (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +# Process Agentless Scanning templates +for template in datadog_agentless_delegate_role.yaml datadog_agentless_scanning.yaml datadog_agentless_delegate_role_snapshot.yaml; do + # Note: unlike main templates, here we remove the 'v' prefix from the version + perl -i.bak -pe "s//${VERSION#v}/g" "$template" + + # Replace ZIPFILE_PLACEHOLDER with the contents of the Python file + perl -i -pe ' + # Read the Python script from stdin + BEGIN { $p = do { local $/; } } + # Find the placeholder and capture its indentation + /^(\s+)/ && ( + # Replace with the Python script, preserving the indentation + $_ = join("\n", map { $1 . $_ } split(/\n/, $p)) . "\n" + ) + ' "$template" < datadog_agentless_api_call.py +done + +trap 'mv datadog_agentless_scanning.yaml.bak datadog_agentless_scanning.yaml; + mv datadog_agentless_delegate_role.yaml.bak datadog_agentless_delegate_role.yaml; + mv datadog_agentless_delegate_role_snapshot.yaml.bak datadog_agentless_delegate_role_snapshot.yaml; +' EXIT + +# Upload +if [ "$PRIVATE_TEMPLATE" = true ] ; then + aws s3 cp . s3://${BUCKET}/aws_agentless/${VERSION} --recursive --exclude "*" --include "*.yaml" +else + aws s3 cp . s3://${BUCKET}/aws_agentless/${VERSION} --recursive --exclude "*" --include "*.yaml" \ + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +fi +echo "Done uploading the agentless templates, and here is the CloudFormation quick launch URL" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-agentless-scanning&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_agentless/${VERSION}/datadog_agentless_scanning.yaml" + +echo "Done!" + diff --git a/aws_agentless/version.txt b/aws_agentless/version.txt new file mode 100644 index 00000000..60453e69 --- /dev/null +++ b/aws_agentless/version.txt @@ -0,0 +1 @@ +v1.0.0 \ No newline at end of file diff --git a/aws_quickstart/release.sh b/aws_quickstart/release.sh index cba4aefa..a9c9d278 100755 --- a/aws_quickstart/release.sh +++ b/aws_quickstart/release.sh @@ -55,28 +55,8 @@ cp main_extended.yaml main_extended.yaml.bak perl -pi -e "s//${BUCKET}/g" main_extended.yaml perl -pi -e "s//${VERSION}/g" main_extended.yaml -# Process Agentless Scanning templates -for template in datadog_agentless_delegate_role.yaml datadog_agentless_scanning.yaml datadog_agentless_delegate_role_snapshot.yaml; do - # Note: unlike above, here we remove the 'v' prefix from the version - perl -i.bak -pe "s//${VERSION#v}/g" "$template" - - # Replace ZIPFILE_PLACEHOLDER with the contents of the Python file - perl -i -pe ' - # Read the Python script from stdin - BEGIN { $p = do { local $/; } } - # Find the placeholder and capture its indentation - /^(\s+)/ && ( - # Replace with the Python script, preserving the indentation - $_ = join("\n", map { $1 . $_ } split(/\n/, $p)) . "\n" - ) - ' "$template" < datadog_agentless_api_call.py -done - trap 'mv main_v2.yaml.bak main_v2.yaml; mv main_extended.yaml.bak main_extended.yaml; - mv datadog_agentless_scanning.yaml.bak datadog_agentless_scanning.yaml; - mv datadog_agentless_delegate_role.yaml.bak datadog_agentless_delegate_role.yaml; - mv datadog_agentless_delegate_role_snapshot.yaml.bak datadog_agentless_delegate_role_snapshot.yaml; ' EXIT # Upload From 2c429c6a0b7b03e2baa42238d686b52cea3a9fcb Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 12:18:47 +0100 Subject: [PATCH 02/10] Pin agentless template version in main_extended --- aws_quickstart/main_extended.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/main_extended.yaml b/aws_quickstart/main_extended.yaml index 3904f649..20e1bf1d 100644 --- a/aws_quickstart/main_extended.yaml +++ b/aws_quickstart/main_extended.yaml @@ -222,8 +222,8 @@ Resources: Properties: TemplateURL: !If - IsCrossAccountScanning - - 'https://.s3.amazonaws.com/aws//datadog_agentless_delegate_role.yaml' - - 'https://.s3.amazonaws.com/aws//datadog_agentless_scanning.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v1.0.0/datadog_agentless_delegate_role.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v1.0.0/datadog_agentless_scanning.yaml' Parameters: DatadogAPIKey: !Ref APIKey DatadogAPPKey: !Ref APPKey From 02d2afd352f506ba5629c978d16655316ed70c36 Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 12:22:30 +0100 Subject: [PATCH 03/10] Add README.md --- aws_agentless/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 aws_agentless/README.md diff --git a/aws_agentless/README.md b/aws_agentless/README.md new file mode 100644 index 00000000..0901de7b --- /dev/null +++ b/aws_agentless/README.md @@ -0,0 +1,30 @@ +# Datadog Agentless Scanning CloudFormation Templates + +This directory contains CloudFormation templates for deploying Datadog Agentless Scanning in AWS accounts. + +Datadog Agentless Scanning enables vulnerability scanning of your AWS resources (hosts, containers, Lambda functions, and sensitive data in S3 buckets) without requiring the installation of agents. + +## Templates + +- **`datadog_agentless_scanning.yaml`**: Main template for deploying the complete Agentless Scanner infrastructure, including EC2 instances, Auto Scaling Groups, IAM roles, and optional VPC resources. +- **`datadog_agentless_delegate_role.yaml`**: Template for creating a delegate role that allows the Agentless Scanner from one account to scan resources in another account (cross-account scanning). +- **`datadog_agentless_delegate_role_snapshot.yaml`**: Template that extends an existing delegate role with permissions to copy snapshots across regions. +- **`datadog_agentless_api_call.py`**: Python Lambda function that registers the Agentless Scanning features with the Datadog API. +- **`datadog_agentless_api_call_test.py`**: Unit tests for the API call Lambda function. + +## Prerequisite## Release Process + +**Important:** These templates must be released to the `datadog-cloudformation-template` S3 bucket before they can be used by customers. + +### Releasing a New Version + +1. Update the version in `version.txt` +2. Run the release script: + ```bash + ./release.sh + ``` + Example: + ```bash + ./release.sh datadog-cloudformation-template + ``` +3. Update the pinned version in `aws_quickstart/main_extended.yaml` \ No newline at end of file From 5d20f67b7dbd1c286cf1848525f60cee1b2e3d81 Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 12:29:28 +0100 Subject: [PATCH 04/10] Release script updates aws_quickstart/main_extended if version is modified --- aws_agentless/README.md | 12 ++++++++++-- aws_agentless/release.sh | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/aws_agentless/README.md b/aws_agentless/README.md index 0901de7b..619967c4 100644 --- a/aws_agentless/README.md +++ b/aws_agentless/README.md @@ -12,7 +12,7 @@ Datadog Agentless Scanning enables vulnerability scanning of your AWS resources - **`datadog_agentless_api_call.py`**: Python Lambda function that registers the Agentless Scanning features with the Datadog API. - **`datadog_agentless_api_call_test.py`**: Unit tests for the API call Lambda function. -## Prerequisite## Release Process +## Release Process **Important:** These templates must be released to the `datadog-cloudformation-template` S3 bucket before they can be used by customers. @@ -27,4 +27,12 @@ Datadog Agentless Scanning enables vulnerability scanning of your AWS resources ```bash ./release.sh datadog-cloudformation-template ``` -3. Update the pinned version in `aws_quickstart/main_extended.yaml` \ No newline at end of file + +The release script will automatically: +- Update the pinned agentless version in `aws_quickstart/main_extended.yaml` to match the new version +- Validate that the version doesn't already exist in S3 +- Replace version placeholders in templates +- Inject the Python Lambda code into the YAML templates +- Upload templates to `s3:///aws_agentless//` + +**Note:** After running the release script, you must commit both the agentless templates AND the updated `aws_quickstart/main_extended.yaml` file to ensure version consistency. \ No newline at end of file diff --git a/aws_agentless/release.sh b/aws_agentless/release.sh index 6c0fb717..4ad6f7d3 100755 --- a/aws_agentless/release.sh +++ b/aws_agentless/release.sh @@ -15,6 +15,28 @@ fi # Read the version VERSION=$(head -n 1 version.txt) +# Update the pinned version in aws_quickstart/main_extended.yaml +MAIN_EXTENDED_PATH="../aws_quickstart/main_extended.yaml" +if [ -f "$MAIN_EXTENDED_PATH" ]; then + echo "⚠️ WARNING: Updating pinned agentless version in aws_quickstart/main_extended.yaml to ${VERSION}" + # Check if the version is already set in main_extended.yaml + if grep -q "aws_agentless/${VERSION}/" "$MAIN_EXTENDED_PATH"; then + echo "✓ Version ${VERSION} is already pinned in main_extended.yaml" + else + # Update both TemplateURL lines that reference aws_agentless versions + sed -i.bak -E "s|(https://datadog-cloudformation-template\.s3\.amazonaws\.com/aws_agentless/)v[0-9]+\.[0-9]+\.[0-9]+/|\1${VERSION}/|g" "$MAIN_EXTENDED_PATH" + if [ -f "${MAIN_EXTENDED_PATH}.bak" ]; then + echo "✓ Updated pinned version in main_extended.yaml (backup saved as main_extended.yaml.bak)" + echo " Please review and commit this change along with the agentless templates" + # Clean up backup file after informing user + rm "${MAIN_EXTENDED_PATH}.bak" + fi + fi +else + echo "⚠️ WARNING: Could not find ${MAIN_EXTENDED_PATH} - skipping version pin update" +fi +echo "" + # Confirm the bucket for the current release doesn't already exist so we don't overwrite it set +e EXIT_CODE=0 From 2b11f9038a173cbc3613e8709f8c9e69fcc3969d Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 12:40:58 +0100 Subject: [PATCH 05/10] Update path in tests --- .github/workflows/python-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index 0191c36d..5c966c96 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -19,5 +19,5 @@ jobs: python-version: "3.13" - name: Run Agentless API Call unit tests run: | - cd aws_quickstart + cd aws_agentless python -B -S -m unittest datadog_agentless_api_call_test.py -v From 5b83674f823ed30148caf049b90d576b30b72797 Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Fri, 31 Oct 2025 17:48:05 +0100 Subject: [PATCH 06/10] tiny text change in release script --- aws_agentless/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_agentless/release.sh b/aws_agentless/release.sh index 4ad6f7d3..0c5f5889 100755 --- a/aws_agentless/release.sh +++ b/aws_agentless/release.sh @@ -18,7 +18,7 @@ VERSION=$(head -n 1 version.txt) # Update the pinned version in aws_quickstart/main_extended.yaml MAIN_EXTENDED_PATH="../aws_quickstart/main_extended.yaml" if [ -f "$MAIN_EXTENDED_PATH" ]; then - echo "⚠️ WARNING: Updating pinned agentless version in aws_quickstart/main_extended.yaml to ${VERSION}" + echo "⚠️ Checking if we need to update pinned agentless version in aws_quickstart/main_extended.yaml to ${VERSION}" # Check if the version is already set in main_extended.yaml if grep -q "aws_agentless/${VERSION}/" "$MAIN_EXTENDED_PATH"; then echo "✓ Version ${VERSION} is already pinned in main_extended.yaml" From ae54f69634a67b03e1c7336189e1d1c4a9c07dde Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Mon, 3 Nov 2025 13:30:39 +0100 Subject: [PATCH 07/10] upgrade version for aws_quickstart --- aws_quickstart/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index fdc089d4..f65ba33c 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v4.1.5 +v4.1.6 From c97d7589e03bee7f38331adf34885b87ebf4fecb Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Mon, 3 Nov 2025 15:09:36 +0100 Subject: [PATCH 08/10] Update aws_agentless/version.txt Co-authored-by: Diogo Pereira --- aws_agentless/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_agentless/version.txt b/aws_agentless/version.txt index 60453e69..8eb38913 100644 --- a/aws_agentless/version.txt +++ b/aws_agentless/version.txt @@ -1 +1 @@ -v1.0.0 \ No newline at end of file +v5.0.0 \ No newline at end of file From 25e15c0944a44b3ae5b66dcba4b9a4dbac2ebf75 Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Mon, 3 Nov 2025 15:10:41 +0100 Subject: [PATCH 09/10] upgrade version for aws_quickstart --- aws_quickstart/main_extended.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/main_extended.yaml b/aws_quickstart/main_extended.yaml index 20e1bf1d..de10b2b1 100644 --- a/aws_quickstart/main_extended.yaml +++ b/aws_quickstart/main_extended.yaml @@ -222,8 +222,8 @@ Resources: Properties: TemplateURL: !If - IsCrossAccountScanning - - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v1.0.0/datadog_agentless_delegate_role.yaml' - - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v1.0.0/datadog_agentless_scanning.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v5.0.0/datadog_agentless_delegate_role.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v5.0.0/datadog_agentless_scanning.yaml' Parameters: DatadogAPIKey: !Ref APIKey DatadogAPPKey: !Ref APPKey From fa7f866845bb8f2a094f09ec5d7a945d590f9c5b Mon Sep 17 00:00:00 2001 From: Moez Ezzeddine Date: Mon, 3 Nov 2025 15:56:32 +0100 Subject: [PATCH 10/10] aws_quickstart/release.sh updates the agentless cfn version placeholder --- aws_quickstart/main_extended.yaml | 4 ++-- aws_quickstart/release.sh | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/main_extended.yaml b/aws_quickstart/main_extended.yaml index de10b2b1..17320fcc 100644 --- a/aws_quickstart/main_extended.yaml +++ b/aws_quickstart/main_extended.yaml @@ -222,8 +222,8 @@ Resources: Properties: TemplateURL: !If - IsCrossAccountScanning - - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v5.0.0/datadog_agentless_delegate_role.yaml' - - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless/v5.0.0/datadog_agentless_scanning.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless//datadog_agentless_delegate_role.yaml' + - 'https://datadog-cloudformation-template.s3.amazonaws.com/aws_agentless//datadog_agentless_scanning.yaml' Parameters: DatadogAPIKey: !Ref APIKey DatadogAPPKey: !Ref APPKey diff --git a/aws_quickstart/release.sh b/aws_quickstart/release.sh index a9c9d278..262d6168 100755 --- a/aws_quickstart/release.sh +++ b/aws_quickstart/release.sh @@ -15,6 +15,9 @@ fi # Read the version VERSION=$(head -n 1 version.txt) +# Read the agentless version +AGENTLESS_VERSION=$(head -n 1 ../aws_agentless/version.txt) + # Confirm the bucket for the current release doesn't already exist so we don't overwrite it set +e EXIT_CODE=0 @@ -54,6 +57,7 @@ perl -pi -e "s//${VERSION}/g" main_v2.yaml cp main_extended.yaml main_extended.yaml.bak perl -pi -e "s//${BUCKET}/g" main_extended.yaml perl -pi -e "s//${VERSION}/g" main_extended.yaml +perl -pi -e "s//${AGENTLESS_VERSION}/g" main_extended.yaml trap 'mv main_v2.yaml.bak main_v2.yaml; mv main_extended.yaml.bak main_extended.yaml;