From 547cba4e24e41b1aa7532cffcde1b79efe646657 Mon Sep 17 00:00:00 2001 From: Kyle Harris Date: Wed, 28 Jan 2026 14:09:14 -0500 Subject: [PATCH 1/3] Add support for prtest07.datadoghq.com datacenter --- aws_account_level_logs/main.yaml | 1 + aws_config_stream/main_config_stream.yaml | 1 + aws_organizations/main_organizations.yaml | 18 +++++++++++++----- aws_organizations/version.txt | 2 +- .../datadog_integration_api_call_v2.yaml | 1 + aws_quickstart/main_extended.yaml | 18 +++++++++++++----- aws_quickstart/main_v2.yaml | 18 +++++++++++++----- aws_quickstart/main_workflow.yaml | 18 +++++++++++++----- aws_quickstart/version.txt | 2 +- aws_streams/streams_single_region.yaml | 2 ++ 10 files changed, 59 insertions(+), 22 deletions(-) diff --git a/aws_account_level_logs/main.yaml b/aws_account_level_logs/main.yaml index 7170cbb8..ee1cae06 100644 --- a/aws_account_level_logs/main.yaml +++ b/aws_account_level_logs/main.yaml @@ -16,6 +16,7 @@ Parameters: - 'https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input' - 'https://aws-kinesis-http-intake.logs.ap1.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose' - 'https://aws-kinesis-http-intake.logs.ap2.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose' + - 'https://aws-kinesis-http-intake.logs.prtest07.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose' - 'https://aws-kinesis-http-intake.logs.us3.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose' - 'https://aws-kinesis-http-intake.logs.us5.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose' - 'https://aws-kinesis-http-intake.logs.ddog-gov.com/v1/input' diff --git a/aws_config_stream/main_config_stream.yaml b/aws_config_stream/main_config_stream.yaml index 37a0d082..939eb99e 100644 --- a/aws_config_stream/main_config_stream.yaml +++ b/aws_config_stream/main_config_stream.yaml @@ -33,6 +33,7 @@ Parameters: - https://cloudplatform-intake.datadoghq.eu/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose - https://cloudplatform-intake.ap1.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose - https://cloudplatform-intake.ap2.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + - https://cloudplatform-intake.prtest07.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose Resources: ConfigurationRecorder: Type: AWS::Config::ConfigurationRecorder diff --git a/aws_organizations/main_organizations.yaml b/aws_organizations/main_organizations.yaml index 4bb3559b..7dc6be43 100644 --- a/aws_organizations/main_organizations.yaml +++ b/aws_organizations/main_organizations.yaml @@ -28,6 +28,7 @@ Parameters: - ddog-gov.com - ap1.datadoghq.com - ap2.datadoghq.com + - prtest07.datadoghq.com IAMRoleName: Description: Customize the name of IAM role for Datadog AWS integration Type: String @@ -88,6 +89,10 @@ Conditions: Fn::Equals: - !Ref DatadogSite - ap2.datadoghq.com + IsPRTEST07: + Fn::Equals: + - !Ref DatadogSite + - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -386,12 +391,15 @@ Resources: - IsAP2 - AWS: !Sub "arn:${AWS::Partition}:iam::412381753143:root" - Fn::If: - - IsGov + - IsPRTEST07 + - AWS: !Sub "arn:${AWS::Partition}:iam::393946873269:root" - Fn::If: - - IsAWSGovCloud - - AWS: !Sub "arn:${AWS::Partition}:iam::065115117704:root" - - AWS: !Sub "arn:${AWS::Partition}:iam::392588925713:root" - - AWS: !Sub "arn:${AWS::Partition}:iam::464622532012:root" + - IsGov + - Fn::If: + - IsAWSGovCloud + - AWS: !Sub "arn:${AWS::Partition}:iam::065115117704:root" + - AWS: !Sub "arn:${AWS::Partition}:iam::392588925713:root" + - AWS: !Sub "arn:${AWS::Partition}:iam::464622532012:root" Action: - "sts:AssumeRole" Condition: diff --git a/aws_organizations/version.txt b/aws_organizations/version.txt index b913b7c6..52cad5ee 100644 --- a/aws_organizations/version.txt +++ b/aws_organizations/version.txt @@ -1 +1 @@ -v4.1.0 +v4.1.1 diff --git a/aws_quickstart/datadog_integration_api_call_v2.yaml b/aws_quickstart/datadog_integration_api_call_v2.yaml index 18d06183..b61fc76b 100644 --- a/aws_quickstart/datadog_integration_api_call_v2.yaml +++ b/aws_quickstart/datadog_integration_api_call_v2.yaml @@ -24,6 +24,7 @@ Parameters: - us5.datadoghq.com - ap1.datadoghq.com - ap2.datadoghq.com + - prtest07.datadoghq.com - ddog-gov.com IAMRoleName: Description: >- diff --git a/aws_quickstart/main_extended.yaml b/aws_quickstart/main_extended.yaml index 3904f649..3fcf1fd0 100644 --- a/aws_quickstart/main_extended.yaml +++ b/aws_quickstart/main_extended.yaml @@ -40,6 +40,7 @@ Parameters: - us5.datadoghq.com - ap1.datadoghq.com - ap2.datadoghq.com + - prtest07.datadoghq.com - ddog-gov.com IAMRoleName: Description: Customize the name of IAM role for Datadog AWS integration @@ -200,6 +201,10 @@ Conditions: Fn::Equals: - !Ref DatadogSite - ap2.datadoghq.com + IsPRTEST07: + Fn::Equals: + - !Ref DatadogSite + - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -264,12 +269,15 @@ Resources: - IsAP2 - "412381753143" - !If - - IsGov + - IsPRTEST07 + - "393946873269" - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsGov + - !If + - IsAWSGovCloud + - "065115117704" + - "392588925713" + - "464622532012" # The Lambda function to ship logs from S3 and CloudWatch, custom metrics and traces from Lambda functions to Datadog # https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring ForwarderStack: diff --git a/aws_quickstart/main_v2.yaml b/aws_quickstart/main_v2.yaml index 88bba2cb..3553ba46 100644 --- a/aws_quickstart/main_v2.yaml +++ b/aws_quickstart/main_v2.yaml @@ -27,6 +27,7 @@ Parameters: - us5.datadoghq.com - ap1.datadoghq.com - ap2.datadoghq.com + - prtest07.datadoghq.com - ddog-gov.com IAMRoleName: Description: Customize the name of IAM role for Datadog AWS integration @@ -104,6 +105,10 @@ Conditions: Fn::Equals: - !Ref DatadogSite - ap2.datadoghq.com + IsPRTEST07: + Fn::Equals: + - !Ref DatadogSite + - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -142,12 +147,15 @@ Resources: - IsAP2 - "412381753143" - !If - - IsGov + - IsPRTEST07 + - "393946873269" - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsGov + - !If + - IsAWSGovCloud + - "065115117704" + - "392588925713" + - "464622532012" # The Lambda function to ship logs from S3 and CloudWatch, custom metrics and traces from Lambda functions to Datadog # https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring ForwarderStack: diff --git a/aws_quickstart/main_workflow.yaml b/aws_quickstart/main_workflow.yaml index 250b2144..8121c967 100644 --- a/aws_quickstart/main_workflow.yaml +++ b/aws_quickstart/main_workflow.yaml @@ -37,6 +37,7 @@ Parameters: - us5.datadoghq.com - ap1.datadoghq.com - ap2.datadoghq.com + - prtest07.datadoghq.com - ddog-gov.com IAMRoleName: Description: Customize the name of IAM role for Datadog AWS integration @@ -93,6 +94,10 @@ Conditions: Fn::Equals: - !Ref DatadogSite - ap2.datadoghq.com + IsPRTEST07: + Fn::Equals: + - !Ref DatadogSite + - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -314,12 +319,15 @@ Resources: - IsAP2 - "412381753143" - !If - - IsGov + - IsPRTEST07 + - "393946873269" - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsGov + - !If + - IsAWSGovCloud + - "065115117704" + - "392588925713" + - "464622532012" # Step 3: Notify IAM role creation finished NotifyIAMRoleCreationFinished: diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index 6062a5eb..db6ecdd8 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v4.3.1 +v4.3.2 diff --git a/aws_streams/streams_single_region.yaml b/aws_streams/streams_single_region.yaml index 7ce59d0c..a71fee61 100644 --- a/aws_streams/streams_single_region.yaml +++ b/aws_streams/streams_single_region.yaml @@ -96,6 +96,8 @@ Mappings: Endpoint: "https://awsmetrics-intake.ap1.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose" "ap2.datadoghq.com": Endpoint: "https://awsmetrics-intake.ap2.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose" + "prtest07.datadoghq.com": + Endpoint: "https://awsmetrics-intake.prtest07.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose" "datadoghq.com": Endpoint: "https://awsmetrics-intake.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose" Resources: From 2678cd54486fd73863773b81378531934d07f074 Mon Sep 17 00:00:00 2001 From: Kyle Harris Date: Wed, 28 Jan 2026 14:42:48 -0500 Subject: [PATCH 2/3] Add centralized datacenter configuration infrastructure (Phase 1) --- .gitignore | 3 + datacenters.yaml | 95 ++++++++++++ scripts/README.md | 86 +++++++++++ scripts/generate_datacenter_configs.py | 151 +++++++++++++++++++ scripts/validate_datacenters_config.py | 192 +++++++++++++++++++++++++ 5 files changed, 527 insertions(+) create mode 100644 datacenters.yaml create mode 100644 scripts/README.md create mode 100755 scripts/generate_datacenter_configs.py create mode 100755 scripts/validate_datacenters_config.py diff --git a/.gitignore b/.gitignore index 33ce3383..57bcad45 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ **/.taskcat/ **/taskcat_outputs/ **/tmp/ + +# Generated datacenter configuration files +generated/ diff --git a/datacenters.yaml b/datacenters.yaml new file mode 100644 index 00000000..a6a1bc68 --- /dev/null +++ b/datacenters.yaml @@ -0,0 +1,95 @@ +# Centralized Datadog Datacenter Configuration +# Single source of truth for all Datadog datacenter endpoints and AWS account IDs +# +# This file is used by scripts/generate_datacenter_configs.py to generate +# CloudFormation template snippets for all datacenter-specific configuration. +# +# When adding a new datacenter, simply add an entry here and regenerate configs. + +version: 1.0 + +datacenters: + - site: datadoghq.com + region: US + account_id: "464622532012" + endpoints: + api: https://api.datadoghq.com + logs_v1: https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input + logs_v2: https://aws-kinesis-http-intake.logs.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: datadoghq.eu + region: EU + account_id: "464622532012" + endpoints: + api: https://api.datadoghq.eu + logs_v1: https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input + logs_v2: https://aws-kinesis-http-intake.logs.datadoghq.eu/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.datadoghq.eu/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.datadoghq.eu/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: us3.datadoghq.com + region: US3 + account_id: "464622532012" + endpoints: + api: https://api.us3.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.us3.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.us3.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.us3.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: us5.datadoghq.com + region: US5 + account_id: "464622532012" + endpoints: + api: https://api.us5.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.us5.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.us5.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.us5.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: ap1.datadoghq.com + region: AP1 + account_id: "417141415827" + endpoints: + api: https://api.ap1.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.ap1.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.ap1.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.ap1.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: ap2.datadoghq.com + region: AP2 + account_id: "412381753143" + endpoints: + api: https://api.ap2.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.ap2.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.ap2.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.ap2.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: prtest07.datadoghq.com + region: PRTEST07 + account_id: "393946873269" + endpoints: + api: https://api.prtest07.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.prtest07.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.prtest07.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.prtest07.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + + - site: ddog-gov.com + region: GOVCLOUD + account_id: "392588925713" # Commercial Gov + account_id_govcloud: "065115117704" # AWS GovCloud partition + endpoints: + api: https://api.ddog-gov.com + logs_v1: https://aws-kinesis-http-intake.logs.ddog-gov.com/v1/input + metrics: https://awsmetrics-intake.ddog-gov.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.ddog-gov.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + +# Test/internal datacenters (for Datadog internal testing only) +test_datacenters: + - site: datad0g.com + region: INTERNAL + account_id: "464622532012" + endpoints: + api: https://api.datad0g.com + logs_v1: https://aws-kinesis-http-intake.logs.datad0g.com/v1/input + metrics: https://awsmetrics-http-intake.datad0g.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..5c821d28 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,86 @@ +# CloudFormation Template Scripts + +This directory contains utility scripts for managing CloudFormation templates. + +## Datacenter Configuration Scripts + +### `generate_datacenter_configs.py` + +Generates CloudFormation configuration snippets from the centralized `datacenters.yaml` file. + +**Usage:** +```bash +python scripts/generate_datacenter_configs.py +``` + +**Outputs:** +- `generated/allowed_values_sites.yaml` - List of all datacenter sites for CloudFormation AllowedValues +- `generated/mappings_account_ids.yaml` - CloudFormation Mappings for AWS account IDs by site +- `generated/mappings_endpoints_logs_v1.yaml` - Logs v1 endpoint mappings +- `generated/mappings_endpoints_logs_v2.yaml` - Logs v2 endpoint mappings +- `generated/mappings_endpoints_metrics.yaml` - Metrics endpoint mappings +- `generated/mappings_endpoints_cloudplatform.yaml` - Config stream endpoint mappings +- `generated/mappings_endpoints_api.yaml` - API endpoint mappings + +**When to run:** +- After adding a new datacenter to `datacenters.yaml` +- Before updating CloudFormation templates with new datacenter support +- As part of the release process (future integration) + +### `validate_datacenters_config.py` + +Validates the structure and content of `datacenters.yaml`. + +**Usage:** +```bash +python scripts/validate_datacenters_config.py +``` + +**Exit codes:** +- `0` - Validation passed +- `1` - Validation failed + +**Validation checks:** +- Required fields are present (site, region, account_id, endpoints) +- Account IDs are valid format (12 digits) +- Endpoint URLs are well-formed HTTPS URLs +- No duplicate datacenter sites +- YAML syntax is valid + +**When to run:** +- After modifying `datacenters.yaml` +- Before committing changes +- As part of CI/CD validation + +## Adding a New Datacenter + +1. Edit `datacenters.yaml` and add a new datacenter entry: + ```yaml + - site: new-site.datadoghq.com + region: NEW_REGION + account_id: "123456789012" + endpoints: + api: https://api.new-site.datadoghq.com + logs_v2: https://aws-kinesis-http-intake.logs.new-site.datadoghq.com/api/v2/logs?dd-protocol=aws-kinesis-firehose + metrics: https://awsmetrics-intake.new-site.datadoghq.com/api/v2/awsmetrics?dd-protocol=aws-kinesis-firehose + cloudplatform: https://cloudplatform-intake.new-site.datadoghq.com/api/v2/cloudchanges?dd-protocol=aws-kinesis-firehose + ``` + +2. Validate the configuration: + ```bash + python scripts/validate_datacenters_config.py + ``` + +3. Generate CloudFormation snippets: + ```bash + python scripts/generate_datacenter_configs.py + ``` + +4. Review generated files in `generated/` directory + +5. (Future) Use generated files to update CloudFormation templates + +## Requirements + +- Python 3.6+ +- PyYAML (`pip install pyyaml`) diff --git a/scripts/generate_datacenter_configs.py b/scripts/generate_datacenter_configs.py new file mode 100755 index 00000000..84cb4e3c --- /dev/null +++ b/scripts/generate_datacenter_configs.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +""" +Generate CloudFormation datacenter configuration from centralized config. + +This script reads datacenters.yaml and generates CloudFormation snippets +for use in templates. Generated files are written to the generated/ directory. + +Usage: + python scripts/generate_datacenter_configs.py + +Outputs: + generated/allowed_values_sites.yaml - List for AllowedValues parameter + generated/mappings_account_ids.yaml - CloudFormation Mappings for account IDs + generated/mappings_endpoints_logs_v1.yaml - Logs v1 endpoint mappings + generated/mappings_endpoints_logs_v2.yaml - Logs v2 endpoint mappings + generated/mappings_endpoints_metrics.yaml - Metrics endpoint mappings + generated/mappings_endpoints_cloudplatform.yaml - Config stream mappings +""" + +import os +import sys +import yaml +from pathlib import Path + + +def load_config(config_path='datacenters.yaml'): + """Load datacenters.yaml configuration file.""" + if not os.path.exists(config_path): + print(f"Error: {config_path} not found", file=sys.stderr) + sys.exit(1) + + with open(config_path, 'r') as f: + return yaml.safe_load(f) + + +def generate_allowed_values(config): + """Generate AllowedValues list for DatadogSite parameter.""" + sites = [dc['site'] for dc in config['datacenters']] + + # Add test datacenters if present + if 'test_datacenters' in config: + sites.extend([dc['site'] for dc in config['test_datacenters']]) + + return sites + + +def generate_account_id_mappings(config): + """Generate CloudFormation Mappings for account IDs.""" + mappings = {'DdAccountIdBySite': {}} + + for dc in config['datacenters']: + entry = {'AccountId': dc['account_id']} + + # Add GovCloud account ID if present + if 'account_id_govcloud' in dc: + entry['AccountIdGovCloud'] = dc['account_id_govcloud'] + + mappings['DdAccountIdBySite'][dc['site']] = entry + + # Add test datacenters + if 'test_datacenters' in config: + for dc in config['test_datacenters']: + mappings['DdAccountIdBySite'][dc['site']] = { + 'AccountId': dc['account_id'] + } + + return {'Mappings': mappings} + + +def generate_endpoint_mappings(config, endpoint_type): + """Generate CloudFormation Mappings for specific endpoint type.""" + # Create mapping name (e.g., logs_v1 -> DdLogsV1EndpointBySite) + mapping_name_parts = endpoint_type.split('_') + mapping_name = 'Dd' + ''.join(p.capitalize() for p in mapping_name_parts) + 'EndpointBySite' + + mappings = {mapping_name: {}} + + for dc in config['datacenters']: + if endpoint_type in dc['endpoints']: + mappings[mapping_name][dc['site']] = { + 'Endpoint': dc['endpoints'][endpoint_type] + } + + # Add test datacenters + if 'test_datacenters' in config: + for dc in config['test_datacenters']: + if endpoint_type in dc['endpoints']: + mappings[mapping_name][dc['site']] = { + 'Endpoint': dc['endpoints'][endpoint_type] + } + + return {'Mappings': mappings} + + +def ensure_generated_dir(): + """Create generated/ directory if it doesn't exist.""" + generated_dir = Path('generated') + generated_dir.mkdir(exist_ok=True) + return generated_dir + + +def write_yaml_file(filepath, data): + """Write data to YAML file.""" + with open(filepath, 'w') as f: + yaml.dump(data, f, default_flow_style=False, sort_keys=False) + + +def main(): + """Main entry point.""" + print("Loading datacenter configuration...") + config = load_config() + + print(f"Found {len(config['datacenters'])} production datacenters") + if 'test_datacenters' in config: + print(f"Found {len(config['test_datacenters'])} test datacenters") + + # Ensure output directory exists + generated_dir = ensure_generated_dir() + + # Generate AllowedValues list + print("\nGenerating AllowedValues list...") + allowed_values = generate_allowed_values(config) + write_yaml_file(generated_dir / 'allowed_values_sites.yaml', allowed_values) + print(f" ✓ Written to generated/allowed_values_sites.yaml ({len(allowed_values)} sites)") + + # Generate account ID mappings + print("\nGenerating account ID mappings...") + account_mappings = generate_account_id_mappings(config) + write_yaml_file(generated_dir / 'mappings_account_ids.yaml', account_mappings) + print(f" ✓ Written to generated/mappings_account_ids.yaml") + + # Generate endpoint mappings for each type + endpoint_types = ['logs_v1', 'logs_v2', 'metrics', 'cloudplatform', 'api'] + + for endpoint_type in endpoint_types: + print(f"\nGenerating {endpoint_type} endpoint mappings...") + endpoint_mappings = generate_endpoint_mappings(config, endpoint_type) + filename = f'mappings_endpoints_{endpoint_type}.yaml' + write_yaml_file(generated_dir / filename, endpoint_mappings) + + # Count how many sites have this endpoint type + count = sum(1 for dc in config['datacenters'] if endpoint_type in dc['endpoints']) + print(f" ✓ Written to generated/{filename} ({count} endpoints)") + + print("\n✅ All datacenter configuration files generated successfully!") + print(f"\nGenerated files are in the 'generated/' directory.") + print("These files can be used to update CloudFormation templates.") + + +if __name__ == '__main__': + main() diff --git a/scripts/validate_datacenters_config.py b/scripts/validate_datacenters_config.py new file mode 100755 index 00000000..389c9763 --- /dev/null +++ b/scripts/validate_datacenters_config.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Validate datacenters.yaml configuration file. + +This script validates the structure and content of datacenters.yaml to ensure: +- Required fields are present +- Account IDs are valid format +- Endpoint URLs are well-formed +- No duplicate sites + +Usage: + python scripts/validate_datacenters_config.py + +Exit codes: + 0 - Validation passed + 1 - Validation failed +""" + +import os +import sys +import yaml +import re +from urllib.parse import urlparse + + +def load_config(config_path='datacenters.yaml'): + """Load and parse datacenters.yaml.""" + if not os.path.exists(config_path): + print(f"❌ Error: {config_path} not found", file=sys.stderr) + return None + + try: + with open(config_path, 'r') as f: + return yaml.safe_load(f) + except yaml.YAMLError as e: + print(f"❌ Error: Invalid YAML syntax: {e}", file=sys.stderr) + return None + + +def validate_account_id(account_id, site): + """Validate AWS account ID format (12 digits).""" + if not isinstance(account_id, str): + print(f"❌ Error: Account ID for {site} must be a string (got {type(account_id).__name__})") + return False + + if not re.match(r'^\d{12}$', account_id): + print(f"❌ Error: Invalid account ID for {site}: '{account_id}' (must be 12 digits)") + return False + + return True + + +def validate_endpoint_url(url, site, endpoint_type): + """Validate endpoint URL format.""" + if not isinstance(url, str): + print(f"❌ Error: Endpoint {endpoint_type} for {site} must be a string") + return False + + if not url.startswith('https://'): + print(f"❌ Error: Endpoint {endpoint_type} for {site} must use HTTPS: {url}") + return False + + try: + parsed = urlparse(url) + if not parsed.netloc: + print(f"❌ Error: Invalid URL for {endpoint_type} endpoint on {site}: {url}") + return False + except Exception as e: + print(f"❌ Error: Failed to parse URL for {endpoint_type} on {site}: {e}") + return False + + return True + + +def validate_datacenter(dc, seen_sites): + """Validate a single datacenter entry.""" + errors = [] + + # Required fields + if 'site' not in dc: + errors.append("Missing required field: 'site'") + return errors + + site = dc['site'] + + # Check for duplicate sites + if site in seen_sites: + errors.append(f"Duplicate site: {site}") + seen_sites.add(site) + + # Validate required fields + if 'region' not in dc: + errors.append(f"Missing 'region' for {site}") + + if 'account_id' not in dc: + errors.append(f"Missing 'account_id' for {site}") + else: + if not validate_account_id(dc['account_id'], site): + errors.append(f"Invalid account_id for {site}") + + # Validate optional GovCloud account ID + if 'account_id_govcloud' in dc: + if not validate_account_id(dc['account_id_govcloud'], f"{site} (GovCloud)"): + errors.append(f"Invalid account_id_govcloud for {site}") + + # Validate endpoints + if 'endpoints' not in dc: + errors.append(f"Missing 'endpoints' for {site}") + else: + endpoints = dc['endpoints'] + if not isinstance(endpoints, dict): + errors.append(f"'endpoints' for {site} must be a dictionary") + else: + # At least one endpoint is required + if len(endpoints) == 0: + errors.append(f"No endpoints defined for {site}") + + # Validate each endpoint URL + for endpoint_type, url in endpoints.items(): + if not validate_endpoint_url(url, site, endpoint_type): + errors.append(f"Invalid {endpoint_type} endpoint for {site}") + + return errors + + +def main(): + """Main validation logic.""" + print("Validating datacenters.yaml...\n") + + config = load_config() + if config is None: + return 1 + + errors = [] + seen_sites = set() + + # Validate top-level structure + if 'version' not in config: + errors.append("Missing 'version' field") + + if 'datacenters' not in config: + errors.append("Missing 'datacenters' field") + print(f"❌ Validation failed with {len(errors)} error(s):") + for error in errors: + print(f" - {error}") + return 1 + + if not isinstance(config['datacenters'], list): + errors.append("'datacenters' must be a list") + print(f"❌ Validation failed with {len(errors)} error(s):") + for error in errors: + print(f" - {error}") + return 1 + + if len(config['datacenters']) == 0: + errors.append("No datacenters defined") + + # Validate each datacenter + print(f"Validating {len(config['datacenters'])} production datacenters...") + for i, dc in enumerate(config['datacenters'], 1): + dc_errors = validate_datacenter(dc, seen_sites) + if dc_errors: + errors.extend([f"Datacenter #{i}: {e}" for e in dc_errors]) + else: + site = dc.get('site', f'#{i}') + print(f" ✓ {site}") + + # Validate test datacenters if present + if 'test_datacenters' in config: + print(f"\nValidating {len(config['test_datacenters'])} test datacenters...") + for i, dc in enumerate(config['test_datacenters'], 1): + dc_errors = validate_datacenter(dc, seen_sites) + if dc_errors: + errors.extend([f"Test datacenter #{i}: {e}" for e in dc_errors]) + else: + site = dc.get('site', f'#{i}') + print(f" ✓ {site}") + + # Print results + print() + if errors: + print(f"❌ Validation failed with {len(errors)} error(s):") + for error in errors: + print(f" - {error}") + return 1 + else: + print("✅ Validation passed! All datacenter configurations are valid.") + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From 0012eaa0e27c9ef819a392b2004bf0a5ae6f8ccc Mon Sep 17 00:00:00 2001 From: Kyle Harris Date: Wed, 28 Jan 2026 15:37:29 -0500 Subject: [PATCH 3/3] Refactor nested conditionals to use CloudFormation Mappings (Phase 2) --- aws_organizations/main_organizations.yaml | 56 +++++++++++------------ aws_quickstart/main_extended.yaml | 50 ++++++++++---------- aws_quickstart/main_v2.yaml | 50 ++++++++++---------- aws_quickstart/main_workflow.yaml | 50 ++++++++++---------- 4 files changed, 100 insertions(+), 106 deletions(-) diff --git a/aws_organizations/main_organizations.yaml b/aws_organizations/main_organizations.yaml index 7dc6be43..8fc14799 100644 --- a/aws_organizations/main_organizations.yaml +++ b/aws_organizations/main_organizations.yaml @@ -76,23 +76,30 @@ Rules: - Ref: CloudSecurityPostureManagement - "true" AssertDescription: CloudSecurityPostureManagement requires ResourceCollection, must enable ResourceCollection +Mappings: + DdAccountIdBySite: + "datadoghq.com": + AccountId: "464622532012" + "datadoghq.eu": + AccountId: "464622532012" + "us3.datadoghq.com": + AccountId: "464622532012" + "us5.datadoghq.com": + AccountId: "464622532012" + "ap1.datadoghq.com": + AccountId: "417141415827" + "ap2.datadoghq.com": + AccountId: "412381753143" + "prtest07.datadoghq.com": + AccountId: "393946873269" + "ddog-gov.com": + AccountId: "392588925713" + AccountIdGovCloud: "065115117704" Conditions: ResourceCollectionPermissions: Fn::Equals: - !Ref DisableResourceCollection - false - IsAP1: - Fn::Equals: - - !Ref DatadogSite - - ap1.datadoghq.com - IsAP2: - Fn::Equals: - - !Ref DatadogSite - - ap2.datadoghq.com - IsPRTEST07: - Fn::Equals: - - !Ref DatadogSite - - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -384,22 +391,15 @@ Resources: Statement: - Effect: Allow Principal: - Fn::If: - - IsAP1 - - AWS: !Sub "arn:${AWS::Partition}:iam::417141415827:root" - - Fn::If: - - IsAP2 - - AWS: !Sub "arn:${AWS::Partition}:iam::412381753143:root" - - Fn::If: - - IsPRTEST07 - - AWS: !Sub "arn:${AWS::Partition}:iam::393946873269:root" - - Fn::If: - - IsGov - - Fn::If: - - IsAWSGovCloud - - AWS: !Sub "arn:${AWS::Partition}:iam::065115117704:root" - - AWS: !Sub "arn:${AWS::Partition}:iam::392588925713:root" - - AWS: !Sub "arn:${AWS::Partition}:iam::464622532012:root" + AWS: !Sub + - "arn:${AWS::Partition}:iam::${AccountId}:root" + - AccountId: !If + - IsGov + - !If + - IsAWSGovCloud + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountIdGovCloud] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] Action: - "sts:AssumeRole" Condition: diff --git a/aws_quickstart/main_extended.yaml b/aws_quickstart/main_extended.yaml index 3fcf1fd0..d36f8e70 100644 --- a/aws_quickstart/main_extended.yaml +++ b/aws_quickstart/main_extended.yaml @@ -165,6 +165,25 @@ Rules: - Ref: AgentlessSensitiveDataScanning - 'true' AssertDescription: Agentless Scanning options require ResourceCollection, must enable ResourceCollection +Mappings: + DdAccountIdBySite: + "datadoghq.com": + AccountId: "464622532012" + "datadoghq.eu": + AccountId: "464622532012" + "us3.datadoghq.com": + AccountId: "464622532012" + "us5.datadoghq.com": + AccountId: "464622532012" + "ap1.datadoghq.com": + AccountId: "417141415827" + "ap2.datadoghq.com": + AccountId: "412381753143" + "prtest07.datadoghq.com": + AccountId: "393946873269" + "ddog-gov.com": + AccountId: "392588925713" + AccountIdGovCloud: "065115117704" Conditions: InstallForwarder: Fn::Equals: @@ -193,18 +212,6 @@ Conditions: - Fn::Equals: - !Ref AgentlessSensitiveDataScanning - true - IsAP1: - Fn::Equals: - - !Ref DatadogSite - - ap1.datadoghq.com - IsAP2: - Fn::Equals: - - !Ref DatadogSite - - ap2.datadoghq.com - IsPRTEST07: - Fn::Equals: - - !Ref DatadogSite - - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -263,21 +270,12 @@ Resources: IAMRoleName: !Ref IAMRoleName ResourceCollectionPermissions: !If [ResourceCollectionPermissions, true, false] DdAWSAccountId: !If - - IsAP1 - - "417141415827" + - IsGov - !If - - IsAP2 - - "412381753143" - - !If - - IsPRTEST07 - - "393946873269" - - !If - - IsGov - - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsAWSGovCloud + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountIdGovCloud] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] # The Lambda function to ship logs from S3 and CloudWatch, custom metrics and traces from Lambda functions to Datadog # https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring ForwarderStack: diff --git a/aws_quickstart/main_v2.yaml b/aws_quickstart/main_v2.yaml index 3553ba46..f1d115c2 100644 --- a/aws_quickstart/main_v2.yaml +++ b/aws_quickstart/main_v2.yaml @@ -88,6 +88,25 @@ Rules: - Ref: CloudSecurityPostureManagement - 'true' AssertDescription: CloudSecurityPostureManagement requires ResourceCollection, must enable ResourceCollection +Mappings: + DdAccountIdBySite: + "datadoghq.com": + AccountId: "464622532012" + "datadoghq.eu": + AccountId: "464622532012" + "us3.datadoghq.com": + AccountId: "464622532012" + "us5.datadoghq.com": + AccountId: "464622532012" + "ap1.datadoghq.com": + AccountId: "417141415827" + "ap2.datadoghq.com": + AccountId: "412381753143" + "prtest07.datadoghq.com": + AccountId: "393946873269" + "ddog-gov.com": + AccountId: "392588925713" + AccountIdGovCloud: "065115117704" Conditions: InstallForwarder: Fn::Equals: @@ -97,18 +116,6 @@ Conditions: Fn::Equals: - !Ref DisableResourceCollection - false - IsAP1: - Fn::Equals: - - !Ref DatadogSite - - ap1.datadoghq.com - IsAP2: - Fn::Equals: - - !Ref DatadogSite - - ap2.datadoghq.com - IsPRTEST07: - Fn::Equals: - - !Ref DatadogSite - - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -141,21 +148,12 @@ Resources: IAMRoleName: !Ref IAMRoleName ResourceCollectionPermissions: !If [ResourceCollectionPermissions, true, false] DdAWSAccountId: !If - - IsAP1 - - "417141415827" + - IsGov - !If - - IsAP2 - - "412381753143" - - !If - - IsPRTEST07 - - "393946873269" - - !If - - IsGov - - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsAWSGovCloud + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountIdGovCloud] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] # The Lambda function to ship logs from S3 and CloudWatch, custom metrics and traces from Lambda functions to Datadog # https://github.com/DataDog/datadog-serverless-functions/tree/master/aws/logs_monitoring ForwarderStack: diff --git a/aws_quickstart/main_workflow.yaml b/aws_quickstart/main_workflow.yaml index 8121c967..47ab48e1 100644 --- a/aws_quickstart/main_workflow.yaml +++ b/aws_quickstart/main_workflow.yaml @@ -74,6 +74,25 @@ Parameters: External ID for the IAM role trust policy. This is generated by the Datadog UI and ensures that only your Datadog organization can assume this role. Type: String +Mappings: + DdAccountIdBySite: + "datadoghq.com": + AccountId: "464622532012" + "datadoghq.eu": + AccountId: "464622532012" + "us3.datadoghq.com": + AccountId: "464622532012" + "us5.datadoghq.com": + AccountId: "464622532012" + "ap1.datadoghq.com": + AccountId: "417141415827" + "ap2.datadoghq.com": + AccountId: "412381753143" + "prtest07.datadoghq.com": + AccountId: "393946873269" + "ddog-gov.com": + AccountId: "392588925713" + AccountIdGovCloud: "065115117704" Conditions: InstallForwarder: Fn::Equals: @@ -86,18 +105,6 @@ Conditions: Fn::Equals: - !Ref DisableResourceCollection - false - IsAP1: - Fn::Equals: - - !Ref DatadogSite - - ap1.datadoghq.com - IsAP2: - Fn::Equals: - - !Ref DatadogSite - - ap2.datadoghq.com - IsPRTEST07: - Fn::Equals: - - !Ref DatadogSite - - prtest07.datadoghq.com IsGov: Fn::Equals: - !Ref DatadogSite @@ -313,21 +320,12 @@ Resources: IAMRoleName: !Ref IAMRoleName ResourceCollectionPermissions: !If [ResourceCollectionPermissions, true, false] DdAWSAccountId: !If - - IsAP1 - - "417141415827" + - IsGov - !If - - IsAP2 - - "412381753143" - - !If - - IsPRTEST07 - - "393946873269" - - !If - - IsGov - - !If - - IsAWSGovCloud - - "065115117704" - - "392588925713" - - "464622532012" + - IsAWSGovCloud + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountIdGovCloud] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] + - !FindInMap [DdAccountIdBySite, !Ref DatadogSite, AccountId] # Step 3: Notify IAM role creation finished NotifyIAMRoleCreationFinished: