From a6a6a8b24c3dfed754c199f8970bb41a14deac30 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 16:46:08 -0400 Subject: [PATCH 01/32] initial implementation --- .../attach_integration_permissions.py | 234 +++++ aws_quickstart/datadog_integration_role.yaml | 986 +++++------------- 2 files changed, 515 insertions(+), 705 deletions(-) create mode 100644 aws_quickstart/attach_integration_permissions.py diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py new file mode 100644 index 00000000..1ce65317 --- /dev/null +++ b/aws_quickstart/attach_integration_permissions.py @@ -0,0 +1,234 @@ +import json +import logging +import hashlib +from urllib.request import Request +import urllib +import cfnresponse +import boto3 + +LOGGER = logging.getLogger() +LOGGER.setLevel(logging.INFO) +API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" +MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document +BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" + +class DatadogAPIError(Exception): + pass + +def get_policy_arn(account_id, policy_name): + """Generate a policy ARN.""" + return f"arn:aws:iam::{account_id}:policy/{policy_name}" + +def create_smart_chunks(permissions): + """Create chunks based on character limit rather than permission count.""" + chunks = [] + current_chunk = [] + + # Base policy structure size (without permissions) + base_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [], + "Resource": "*" + } + ] + } + base_size = len(json.dumps(base_policy, separators=(',', ':'))) + + for permission in permissions: + # Calculate size if we add this permission + test_chunk = current_chunk + [permission] + test_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": test_chunk, + "Resource": "*" + } + ] + } + test_size = len(json.dumps(test_policy, separators=(',', ':'))) + + # If adding this permission would exceed the limit, start a new chunk + if test_size > MAX_POLICY_SIZE and current_chunk: + chunks.append(current_chunk) + current_chunk = [permission] + current_size = len(json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [permission], + "Resource": "*" + } + ] + }, separators=(',', ':'))) + else: + current_chunk.append(permission) + current_size = test_size + + # Add the last chunk if it has permissions + if current_chunk: + chunks.append(current_chunk) + + return chunks + +def fetch_permissions_from_datadog(): + """Fetch permissions from Datadog API.""" + api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" + headers = { + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = Request(api_url, headers=headers) + request.get_method = lambda: "GET" + + response = urllib.request.urlopen(request) + json_response = json.loads(response.read()) + if response.getcode() != 200: + error_message = json_response.get('errors', ['Unknown error'])[0] + raise DatadogAPIError(f"Datadog API error: {error_message}") + + return json_response["data"]["attributes"]["permissions"] + +def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name, max_policies=20): + """Clean up existing policies with both old and new naming patterns""" + + # Clean up new dynamic policies (datadog-aws-integration-iam-permissions-part{N}) + for i in range(max_policies): + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_arn = get_policy_arn(account_id, policy_name) + try: + iam_client.detach_role_policy( + RoleName=role_name, + PolicyArn=policy_arn + ) + except iam_client.exceptions.NoSuchEntityException: + # Policy to detach doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error detaching policy {policy_name}: {str(e)}") + + try: + iam_client.delete_policy( + PolicyArn=policy_arn + ) + except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): + # Policy to delete doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + + # Clean up old hardcoded managed policies ({IAMRoleName}-ManagedPolicy-{N}) + # Extract role name from the base_policy_name for old policy cleanup + role_name_from_hash = role_name # We have the role name directly + for i in range(1, 5): # Old template had ManagedPolicy-1 through ManagedPolicy-4 + old_policy_name = f"{role_name_from_hash}-ManagedPolicy-{i}" + old_policy_arn = get_policy_arn(account_id, old_policy_name) + try: + iam_client.detach_role_policy( + RoleName=role_name, + PolicyArn=old_policy_arn + ) + LOGGER.info(f"Detached old policy: {old_policy_name}") + except iam_client.exceptions.NoSuchEntityException: + # Policy to detach doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error detaching old policy {old_policy_name}: {str(e)}") + + try: + iam_client.delete_policy( + PolicyArn=old_policy_arn + ) + LOGGER.info(f"Deleted old policy: {old_policy_name}") + except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): + # Policy to delete doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") + +def handle_delete(event, context, role_name, account_id, base_policy_name): + """Handle stack deletion.""" + iam_client = boto3.client('iam') + try: + cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error deleting policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + +def handle_create_update(event, context, role_name, account_id, base_policy_name): + """Handle stack creation or update.""" + try: + # Fetch and smart chunk permissions based on character limits + permissions = fetch_permissions_from_datadog() + permission_chunks = create_smart_chunks(permissions) + + LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # Clean up existing policies + iam_client = boto3.client('iam') + cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + + # Create and attach new policies + for i, chunk in enumerate(permission_chunks): + # Create policy + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": chunk, + "Resource": "*" + } + ] + } + + # Verify policy size before creating + policy_json = json.dumps(policy_document, separators=(',', ':')) + policy_size = len(policy_json) + LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + + if policy_size > MAX_POLICY_SIZE: + LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + + policy = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=policy_json + ) + + # Attach policy to role + iam_client.attach_role_policy( + RoleName=role_name, + PolicyArn=policy['Policy']['Arn'] + ) + + # probably not needed######################################################## + # # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled + # if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": + # iam_client.attach_role_policy( + # RoleName=role_name, + # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) + # ) + ######################################################### + + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error creating/attaching policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + +def handler(event, context): + LOGGER.info("Event received: %s", json.dumps(event)) + + role_name = event['ResourceProperties']['DatadogIntegrationRole'] + account_id = event['ResourceProperties']['AccountId'] + + if event['RequestType'] == 'Delete': + handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + else: + handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 84488aa9..78f60e76 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -143,9 +143,9 @@ Resources: - "organizations:List*" - "rds:Describe*" - "rds:List*" + - "redshift-serverless:ListNamespaces" - "redshift:DescribeClusters" - "redshift:DescribeLoggingStatus" - - "redshift-serverless:ListNamespaces" - "route53:List*" - "route53resolver:ListResolverQueryLogConfigs" - "s3:GetBucketLocation" @@ -173,719 +173,295 @@ Resources: - "wafv2:ListLoggingConfigurations" - "xray:BatchGetTraces" - "xray:GetTraceSummaries" - DatadogIntegrationRoleManagedPolicy1: - Type: "AWS::IAM::ManagedPolicy" + # Lambda function to dynamically attach permissions from DataDog API + DatadogAttachIntegrationPermissionsLambdaExecutionRole: + Type: AWS::IAM::Role Condition: ShouldInstallSecurityAuditPolicy Properties: - ManagedPolicyName: !Sub - - "${IAMRoleName}-ManagedPolicy-1" - - { IAMRoleName: !Ref IAMRoleName } - Roles: - - !Ref DatadogIntegrationRole - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Resource: "*" - Action: - - "account:GetContactInformation" - - "amplify:ListApps" - - "amplify:ListArtifacts" - - "amplify:ListBackendEnvironments" - - "amplify:ListBranches" - - "amplify:ListDomainAssociations" - - "amplify:ListJobs" - - "amplify:ListWebhooks" - - "aoss:BatchGetCollection" - - "aoss:ListCollections" - - "app-integrations:GetApplication" - - "app-integrations:GetDataIntegration" - - "app-integrations:ListApplicationAssociations" - - "app-integrations:ListApplications" - - "app-integrations:ListDataIntegrationAssociations" - - "app-integrations:ListDataIntegrations" - - "app-integrations:ListEventIntegrationAssociations" - - "app-integrations:ListEventIntegrations" - - "appstream:DescribeAppBlockBuilders" - - "appstream:DescribeAppBlocks" - - "appstream:DescribeApplications" - - "appstream:DescribeFleets" - - "appstream:DescribeImageBuilders" - - "appstream:DescribeImages" - - "appstream:DescribeStacks" - - "appsync:GetGraphqlApi" - - "aps:DescribeRuleGroupsNamespace" - - "aps:DescribeScraper" - - "aps:DescribeWorkspace" - - "aps:ListRuleGroupsNamespaces" - - "aps:ListScrapers" - - "aps:ListWorkspaces" - - "athena:BatchGetNamedQuery" - - "athena:BatchGetPreparedStatement" - - "auditmanager:GetAssessment" - - "auditmanager:GetAssessmentFramework" - - "auditmanager:GetControl" - - "b2bi:GetCapability" - - "b2bi:GetPartnership" - - "b2bi:GetProfile" - - "b2bi:GetTransformer" - - "b2bi:ListCapabilities" - - "b2bi:ListPartnerships" - - "b2bi:ListProfiles" - - "b2bi:ListTransformers" - - "backup-gateway:GetGateway" - - "backup-gateway:GetHypervisor" - - "backup-gateway:GetVirtualMachine" - - "backup-gateway:ListGateways" - - "backup-gateway:ListHypervisors" - - "backup-gateway:ListVirtualMachines" - - "backup:DescribeFramework" - - "backup:GetLegalHold" - - "backup:ListBackupPlans" - - "backup:ListFrameworks" - - "backup:ListLegalHolds" - - "backup:ListProtectedResources" - - "backup:ListRecoveryPointsByBackupVault" - - "batch:DescribeJobQueues" - - "batch:DescribeSchedulingPolicies" - - "batch:ListSchedulingPolicies" - - "bedrock:GetAgent" - - "bedrock:GetAgentActionGroup" - - "bedrock:GetAsyncInvoke" - - "bedrock:GetBlueprint" - - "bedrock:GetDataSource" - - "bedrock:GetEvaluationJob" - - "bedrock:GetFlow" - - "bedrock:GetFlowVersion" - - "bedrock:GetGuardrail" - - "bedrock:GetKnowledgeBase" - - "bedrock:GetModelInvocationJob" - - "bedrock:GetPrompt" - - "bedrock:ListAgentCollaborators" - - "bedrock:ListAsyncInvokes" - - "bedrock:ListBlueprints" - - "bedrock:ListKnowledgeBaseDocuments" - - "cassandra:Select" - - "ce:DescribeCostCategoryDefinition" - - "ce:GetAnomalyMonitors" - - "ce:GetAnomalySubscriptions" - - "ce:GetCostCategories" - - "cloudformation:DescribeGeneratedTemplate" - - "cloudformation:DescribeResourceScan" - - "cloudformation:ListGeneratedTemplates" - - "cloudformation:ListResourceScans" - - "cloudformation:ListTypes" - - "cloudhsm:DescribeBackups" - - "cloudhsm:DescribeClusters" - - "codeartifact:DescribeDomain" - - "codeartifact:DescribePackageGroup" - - "codeartifact:DescribeRepository" - - "codeartifact:ListDomains" - - "codeartifact:ListPackageGroups" - - "codeartifact:ListPackages" - - "codeguru-profiler:ListFindingsReports" - - "codeguru-profiler:ListProfilingGroups" - - "codeguru-reviewer:ListCodeReviews" - - "codeguru-reviewer:ListRepositoryAssociations" - - "codeguru-security:GetFindings" - - "codeguru-security:GetScan" - - "codeguru-security:ListScans" - - "codepipeline:GetActionType" - - "codepipeline:ListActionTypes" - - "codepipeline:ListWebhooks" - - "connect:DescribeAgentStatus" - - "connect:DescribeAuthenticationProfile" - - "connect:DescribeContactFlow" - - "connect:DescribeContactFlowModule" - - "connect:DescribeHoursOfOperation" - - "connect:DescribeInstance" - - "connect:DescribeQueue" - - "connect:DescribeQuickConnect" - - "connect:DescribeRoutingProfile" - - "connect:DescribeSecurityProfile" - - "connect:DescribeUser" - - "connect:ListAgentStatuses" - - "connect:ListAuthenticationProfiles" - - "connect:ListContactFlowModules" - - "connect:ListContactFlows" - - "connect:ListHoursOfOperations" - - "connect:ListQueues" - - "connect:ListQuickConnects" - - "connect:ListRoutingProfiles" - - "connect:ListSecurityProfiles" - - "connect:ListUsers" - - "controltower:GetLandingZone" - - "controltower:ListEnabledBaselines" - - "controltower:ListEnabledControls" - - "controltower:ListLandingZones" - - "databrew:ListDatasets" - - "databrew:ListRecipes" - - "databrew:ListRulesets" - - "databrew:ListSchedules" - - "datazone:GetDomain" - - "datazone:ListDomains" - - "deadline:GetBudget" - - "deadline:GetLicenseEndpoint" - - "deadline:GetQueue" - - "deadline:ListBudgets" - - "deadline:ListFarms" - - "deadline:ListFleets" - - "deadline:ListLicenseEndpoints" - - "deadline:ListMonitors" - - "deadline:ListQueues" - - "deadline:ListWorkers" - - "devicefarm:ListDeviceInstances" - - "devicefarm:ListDevicePools" - - "devicefarm:ListDevices" - - "devicefarm:ListInstanceProfiles" - - "devicefarm:ListNetworkProfiles" - - "devicefarm:ListRemoteAccessSessions" - - "devicefarm:ListTestGridProjects" - - "devicefarm:ListTestGridSessions" - - "devicefarm:ListUploads" - - "devicefarm:ListVPCEConfigurations" - - "dlm:GetLifecyclePolicies" - - "dlm:GetLifecyclePolicy" - - "docdb-elastic:GetCluster" - - "docdb-elastic:GetClusterSnapshot" - - "docdb-elastic:ListClusterSnapshots" - - "drs:DescribeJobs" - - "drs:DescribeLaunchConfigurationTemplates" - - "drs:DescribeRecoveryInstances" - - "drs:DescribeReplicationConfigurationTemplates" - - "drs:DescribeSourceNetworks" - - "drs:DescribeSourceServers" - - "dsql:GetCluster" - - "dsql:ListClusters" - - "dynamodb:DescribeBackup" - - "dynamodb:DescribeStream" - - "ec2:GetAllowedImagesSettings" - - "ec2:GetEbsDefaultKmsKeyId" - - "ec2:GetInstanceMetadataDefaults" - - "ec2:GetSerialConsoleAccessStatus" - - "ec2:GetSnapshotBlockPublicAccessState" - - "ec2:GetVerifiedAccessEndpointPolicy" - - "ec2:GetVerifiedAccessEndpointTargets" - - "ec2:GetVerifiedAccessGroupPolicy" - - "eks:DescribeAccessEntry" - - "eks:DescribeAddon" - - "eks:DescribeIdentityProviderConfig" - - "eks:DescribeInsight" - - "eks:DescribePodIdentityAssociation" - - "eks:DescribeUpdate" - - "eks:ListAccessEntries" - - "eks:ListAddons" - - "eks:ListAssociatedAccessPolicies" - - "eks:ListEksAnywhereSubscriptions" - DatadogIntegrationRoleManagedPolicy2: - Type: "AWS::IAM::ManagedPolicy" - Condition: ShouldInstallSecurityAuditPolicy - Properties: - ManagedPolicyName: !Sub - - "${IAMRoleName}-ManagedPolicy-2" - - { IAMRoleName: !Ref IAMRoleName } - Roles: - - !Ref DatadogIntegrationRole - PolicyDocument: - Version: 2012-10-17 + AssumeRolePolicyDocument: + Version: '2012-10-17' Statement: - Effect: Allow - Resource: "*" + Principal: + Service: + - lambda.amazonaws.com Action: - - "eks:ListIdentityProviderConfigs" - - "eks:ListInsights" - - "eks:ListPodIdentityAssociations" - - "elasticmapreduce:ListInstanceFleets" - - "elasticmapreduce:ListInstanceGroups" - - "emr-containers:ListManagedEndpoints" - - "emr-containers:ListSecurityConfigurations" - - "emr-containers:ListVirtualClusters" - - "frauddetector:DescribeDetector" - - "frauddetector:DescribeModelVersions" - - "frauddetector:GetBatchImportJobs" - - "frauddetector:GetBatchPredictionJobs" - - "frauddetector:GetDetectorVersion" - - "frauddetector:GetEntityTypes" - - "frauddetector:GetEventTypes" - - "frauddetector:GetExternalModels" - - "frauddetector:GetLabels" - - "frauddetector:GetListsMetadata" - - "frauddetector:GetModels" - - "frauddetector:GetOutcomes" - - "frauddetector:GetRules" - - "frauddetector:GetVariables" - - "gamelift:DescribeGameSessionQueues" - - "gamelift:DescribeMatchmakingConfigurations" - - "gamelift:DescribeMatchmakingRuleSets" - - "gamelift:ListAliases" - - "gamelift:ListContainerFleets" - - "gamelift:ListContainerGroupDefinitions" - - "gamelift:ListGameServerGroups" - - "gamelift:ListLocations" - - "gamelift:ListScripts" - - "geo:DescribeGeofenceCollection" - - "geo:DescribeKey" - - "geo:DescribeMap" - - "geo:DescribePlaceIndex" - - "geo:DescribeRouteCalculator" - - "geo:DescribeTracker" - - "geo:ListGeofenceCollections" - - "geo:ListKeys" - - "geo:ListPlaceIndexes" - - "geo:ListRouteCalculators" - - "geo:ListTrackers" - - "glacier:GetVaultNotifications" - - "glue:ListRegistries" - - "grafana:DescribeWorkspace" - - "greengrass:GetBulkDeploymentStatus" - - "greengrass:GetComponent" - - "greengrass:GetConnectivityInfo" - - "greengrass:GetCoreDevice" - - "greengrass:GetDeployment" - - "greengrass:GetGroup" - - "imagebuilder:GetContainerRecipe" - - "imagebuilder:GetDistributionConfiguration" - - "imagebuilder:GetImageRecipe" - - "imagebuilder:GetInfrastructureConfiguration" - - "imagebuilder:GetLifecyclePolicy" - - "imagebuilder:GetWorkflow" - - "imagebuilder:ListComponents" - - "imagebuilder:ListContainerRecipes" - - "imagebuilder:ListDistributionConfigurations" - - "imagebuilder:ListImagePipelines" - - "imagebuilder:ListImageRecipes" - - "imagebuilder:ListImages" - - "imagebuilder:ListInfrastructureConfigurations" - - "imagebuilder:ListLifecyclePolicies" - - "imagebuilder:ListWorkflows" - - "iotfleetwise:GetCampaign" - - "iotfleetwise:GetSignalCatalog" - - "iotfleetwise:GetStateTemplate" - - "iotfleetwise:GetVehicle" - - "iotfleetwise:ListCampaigns" - - "iotfleetwise:ListDecoderManifests" - - "iotfleetwise:ListFleets" - - "iotfleetwise:ListSignalCatalogs" - - "iotfleetwise:ListStateTemplates" - - "iotfleetwise:ListVehicles" - - "iotsitewise:DescribeAsset" - - "iotsitewise:DescribeAssetModel" - - "iotsitewise:DescribeDashboard" - - "iotsitewise:DescribeDataset" - - "iotsitewise:DescribePortal" - - "iotsitewise:DescribeProject" - - "iotsitewise:ListAssets" - - "iotsitewise:ListDashboards" - - "iotsitewise:ListDatasets" - - "iotsitewise:ListPortals" - - "iotsitewise:ListProjects" - - "iotsitewise:ListTimeSeries" - - "iottwinmaker:GetComponentType" - - "iottwinmaker:GetEntity" - - "iottwinmaker:GetScene" - - "iottwinmaker:GetWorkspace" - - "iottwinmaker:ListComponentTypes" - - "iottwinmaker:ListEntities" - - "iottwinmaker:ListScenes" - - "iotwireless:GetDeviceProfile" - - "iotwireless:GetMulticastGroup" - - "iotwireless:GetNetworkAnalyzerConfiguration" - - "iotwireless:GetServiceProfile" - - "iotwireless:GetWirelessDevice" - - "iotwireless:GetWirelessGateway" - - "iotwireless:ListDestinations" - - "iotwireless:ListDeviceProfiles" - - "iotwireless:ListMulticastGroups" - - "iotwireless:ListNetworkAnalyzerConfigurations" - - "iotwireless:ListServiceProfiles" - - "iotwireless:ListWirelessDevices" - - "iotwireless:ListWirelessGateways" - - "ivs:GetChannel" - - "ivs:GetComposition" - - "ivs:GetEncoderConfiguration" - - "ivs:GetIngestConfiguration" - - "ivs:GetPublicKey" - - "ivs:GetRecordingConfiguration" - - "ivs:GetStage" - - "ivs:ListChannels" - - "ivs:ListCompositions" - - "ivs:ListEncoderConfigurations" - - "ivs:ListIngestConfigurations" - - "ivs:ListPlaybackKeyPairs" - - "ivs:ListPlaybackRestrictionPolicies" - - "ivs:ListPublicKeys" - - "ivs:ListRecordingConfigurations" - - "ivs:ListStages" - - "ivs:ListStorageConfigurations" - - "ivs:ListStreamKeys" - - "ivschat:GetLoggingConfiguration" - - "ivschat:GetRoom" - - "ivschat:ListLoggingConfigurations" - - "ivschat:ListRooms" - - "lakeformation:GetDataLakeSettings" - - "lakeformation:ListPermissions" - - "lambda:GetFunction" - - "launchwizard:GetDeployment" - - "launchwizard:ListDeployments" - - "lightsail:GetAlarms" - - "lightsail:GetCertificates" - - "lightsail:GetDistributions" - - "lightsail:GetInstancePortStates" - - "lightsail:GetRelationalDatabaseParameters" - - "lightsail:GetRelationalDatabaseSnapshots" - - "lightsail:GetRelationalDatabases" - - "lightsail:GetStaticIps" - - "macie2:GetAllowList" - - "macie2:GetCustomDataIdentifier" - - "macie2:GetMacieSession" - - "macie2:ListAllowLists" - - "macie2:ListCustomDataIdentifiers" - - "macie2:ListMembers" - - "managedblockchain:GetAccessor" - - "managedblockchain:GetMember" - - "managedblockchain:GetNetwork" - - "managedblockchain:GetNode" - - "managedblockchain:GetProposal" - - "managedblockchain:ListAccessors" - - "managedblockchain:ListInvitations" - - "managedblockchain:ListMembers" - - "managedblockchain:ListNodes" - - "managedblockchain:ListProposals" - - "medialive:ListChannelPlacementGroups" - - "medialive:ListCloudWatchAlarmTemplateGroups" - - "medialive:ListCloudWatchAlarmTemplates" - - "medialive:ListClusters" - - "medialive:ListEventBridgeRuleTemplateGroups" - - "medialive:ListEventBridgeRuleTemplates" - - "medialive:ListInputDevices" - - "medialive:ListInputSecurityGroups" - - "medialive:ListInputs" - - "medialive:ListMultiplexes" - - "medialive:ListNetworks" - - "medialive:ListNodes" - - "medialive:ListOfferings" - - "medialive:ListReservations" - - "medialive:ListSdiSources" - - "medialive:ListSignalMaps" - - "mediapackage-vod:DescribeAsset" - - "mediapackage-vod:ListAssets" - - "mediapackage-vod:ListPackagingConfigurations" - - "mediapackage:ListChannels" - - "mediapackage:ListHarvestJobs" - - "mediapackagev2:GetChannel" - DatadogIntegrationRoleManagedPolicy3: - Type: "AWS::IAM::ManagedPolicy" + - sts:AssumeRole + Path: "/" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + Policies: + - PolicyName: !Sub "datadog-aws-integration-iam-permissions-${IAMRoleName}" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - iam:CreatePolicy + - iam:DeletePolicy + - iam:AttachRolePolicy + - iam:DetachRolePolicy + Resource: + - !Sub arn:aws:iam::${AWS::AccountId}:role/${IAMRoleName} + - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-iam-permissions-* + - !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" + DatadogAttachIntegrationPermissionsFunction: + Type: AWS::Lambda::Function Condition: ShouldInstallSecurityAuditPolicy Properties: - ManagedPolicyName: !Sub - - "${IAMRoleName}-ManagedPolicy-3" - - { IAMRoleName: !Ref IAMRoleName } - Roles: - - !Ref DatadogIntegrationRole - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Resource: "*" - Action: - - "mediapackagev2:GetChannelGroup" - - "mediapackagev2:GetChannelPolicy" - - "mediapackagev2:GetOriginEndpoint" - - "mediapackagev2:GetOriginEndpointPolicy" - - "mediapackagev2:ListChannelGroups" - - "mediapackagev2:ListChannels" - - "mediapackagev2:ListHarvestJobs" - - "mediapackagev2:ListOriginEndpoints" - - "memorydb:DescribeAcls" - - "memorydb:DescribeMultiRegionClusters" - - "memorydb:DescribeParameterGroups" - - "memorydb:DescribeReservedNodes" - - "memorydb:DescribeSnapshots" - - "memorydb:DescribeSubnetGroups" - - "memorydb:DescribeUsers" - - "mobiletargeting:GetApps" - - "mobiletargeting:GetCampaigns" - - "mobiletargeting:GetChannels" - - "mobiletargeting:GetEventStream" - - "mobiletargeting:GetSegments" - - "mobiletargeting:ListJourneys" - - "mobiletargeting:ListTemplates" - - "network-firewall:DescribeTLSInspectionConfiguration" - - "network-firewall:DescribeVpcEndpointAssociation" - - "network-firewall:ListTLSInspectionConfigurations" - - "network-firewall:ListVpcEndpointAssociations" - - "networkmanager:GetConnectPeer" - - "networkmanager:GetConnections" - - "networkmanager:GetCoreNetwork" - - "networkmanager:GetDevices" - - "networkmanager:GetLinks" - - "networkmanager:GetSites" - - "networkmanager:ListAttachments" - - "networkmanager:ListConnectPeers" - - "networkmanager:ListCoreNetworks" - - "networkmanager:ListPeerings" - - "osis:GetPipeline" - - "osis:GetPipelineBlueprint" - - "osis:ListPipelineBlueprints" - - "osis:ListPipelines" - - "payment-cryptography:GetKey" - - "payment-cryptography:ListAliases" - - "payment-cryptography:ListKeys" - - "pca-connector-ad:ListConnectors" - - "pca-connector-ad:ListDirectoryRegistrations" - - "pca-connector-ad:ListTemplates" - - "pca-connector-scep:ListConnectors" - - "personalize:DescribeAlgorithm" - - "personalize:DescribeBatchInferenceJob" - - "personalize:DescribeBatchSegmentJob" - - "personalize:DescribeCampaign" - - "personalize:DescribeDataDeletionJob" - - "personalize:DescribeDataset" - - "personalize:DescribeDatasetExportJob" - - "personalize:DescribeDatasetImportJob" - - "personalize:DescribeEventTracker" - - "personalize:DescribeFeatureTransformation" - - "personalize:DescribeFilter" - - "personalize:DescribeMetricAttribution" - - "personalize:DescribeRecipe" - - "personalize:DescribeRecommender" - - "personalize:DescribeSchema" - - "personalize:DescribeSolution" - - "personalize:ListBatchInferenceJobs" - - "personalize:ListBatchSegmentJobs" - - "personalize:ListCampaigns" - - "personalize:ListDataDeletionJobs" - - "personalize:ListDatasetExportJobs" - - "personalize:ListDatasetImportJobs" - - "personalize:ListDatasets" - - "personalize:ListEventTrackers" - - "personalize:ListFilters" - - "personalize:ListMetricAttributions" - - "personalize:ListRecipes" - - "personalize:ListRecommenders" - - "personalize:ListSchemas" - - "personalize:ListSolutions" - - "pipes:ListPipes" - - "proton:GetComponent" - - "proton:GetDeployment" - - "proton:GetEnvironment" - - "proton:GetEnvironmentAccountConnection" - - "proton:GetEnvironmentTemplate" - - "proton:GetEnvironmentTemplateVersion" - - "proton:GetRepository" - - "proton:GetService" - - "proton:GetServiceInstance" - - "proton:GetServiceTemplate" - - "proton:GetServiceTemplateVersion" - - "proton:ListComponents" - - "proton:ListDeployments" - - "proton:ListEnvironmentAccountConnections" - - "proton:ListEnvironmentTemplateVersions" - - "proton:ListEnvironmentTemplates" - - "proton:ListEnvironments" - - "proton:ListRepositories" - - "proton:ListServiceInstances" - - "proton:ListServiceTemplateVersions" - - "proton:ListServiceTemplates" - - "proton:ListServices" - - "qbusiness:GetApplication" - - "qbusiness:GetDataAccessor" - - "qbusiness:GetDataSource" - - "qbusiness:GetIndex" - - "qbusiness:GetPlugin" - - "qbusiness:GetRetriever" - - "qbusiness:GetWebExperience" - - "qbusiness:ListDataAccessors" - - "qldb:ListJournalKinesisStreamsForLedger" - - "ram:GetResourceShareInvitations" - - "rbin:GetRule" - - "rbin:ListRules" - - "redshift-serverless:GetSnapshot" - - "redshift-serverless:ListEndpointAccess" - - "redshift-serverless:ListManagedWorkgroups" - - "redshift-serverless:ListNamespaces" - - "redshift-serverless:ListRecoveryPoints" - - "redshift-serverless:ListSnapshots" - - "refactor-spaces:ListApplications" - - "refactor-spaces:ListEnvironments" - - "refactor-spaces:ListRoutes" - - "refactor-spaces:ListServices" - - "resiliencehub:DescribeApp" - - "resiliencehub:DescribeAppAssessment" - - "resiliencehub:ListAppAssessments" - - "resiliencehub:ListApps" - - "resiliencehub:ListResiliencyPolicies" - - "resource-explorer-2:GetIndex" - - "resource-explorer-2:GetManagedView" - - "resource-explorer-2:GetView" - - "resource-explorer-2:ListManagedViews" - - "resource-explorer-2:ListViews" - - "resource-groups:GetGroup" - - "resource-groups:ListGroups" - - "route53-recovery-readiness:ListCells" - - "route53-recovery-readiness:ListReadinessChecks" - - "route53-recovery-readiness:ListRecoveryGroups" - - "route53-recovery-readiness:ListResourceSets" - - "rum:GetAppMonitor" - - "rum:ListAppMonitors" - - "s3-outposts:ListRegionalBuckets" - - "savingsplans:DescribeSavingsPlanRates" - - "savingsplans:DescribeSavingsPlans" - - "scheduler:GetSchedule" - - "scheduler:ListScheduleGroups" - - "scheduler:ListSchedules" - - "securitylake:ListDataLakes" - - "securitylake:ListSubscribers" - - "servicecatalog:DescribePortfolio" - - "servicecatalog:DescribeProduct" - - "servicecatalog:GetApplication" - - "servicecatalog:GetAttributeGroup" - - "servicecatalog:ListApplications" - - "servicecatalog:ListAttributeGroups" - - "servicecatalog:ListPortfolios" - - "servicecatalog:SearchProducts" - - "servicediscovery:GetNamespace" - - "servicediscovery:GetService" - - "servicediscovery:ListNamespaces" - - "servicediscovery:ListServices" - - "ses:GetArchive" - - "ses:GetContactList" - - "ses:GetCustomVerificationEmailTemplate" - - "ses:GetDedicatedIpPool" - - "ses:GetIdentityMailFromDomainAttributes" - - "ses:GetIngressPoint" - - "ses:GetMultiRegionEndpoint" - - "ses:GetRelay" - - "ses:GetRuleSet" - - "ses:GetTemplate" - - "ses:GetTrafficPolicy" - - "ses:ListAddonInstances" - - "ses:ListAddonSubscriptions" - - "ses:ListAddressLists" - - "ses:ListArchives" - - "ses:ListContactLists" - DatadogIntegrationRoleManagedPolicy4: - Type: "AWS::IAM::ManagedPolicy" + Description: "A function to attach Datadog AWS integration permissions to an IAM role." + Role: !GetAtt DatadogAttachIntegrationPermissionsLambdaExecutionRole.Arn + Handler: "index.handler" + LoggingConfig: + ApplicationLogLevel: "INFO" + LogFormat: "JSON" + Runtime: "python3.11" + Timeout: 300 + Code: + ZipFile: | + import json + import logging + import hashlib + from urllib.request import Request + import urllib + import cfnresponse + import boto3 + + LOGGER = logging.getLogger() + LOGGER.setLevel(logging.INFO) + API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" + MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document + BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" + + class DatadogAPIError(Exception): + pass + + def get_policy_arn(account_id, policy_name): + """Generate a policy ARN.""" + return f"arn:aws:iam::{account_id}:policy/{policy_name}" + + def create_smart_chunks(permissions): + """Create chunks based on character limit rather than permission count.""" + chunks = [] + current_chunk = [] + current_size = 0 + + # Base policy structure size (without permissions) + base_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [], + "Resource": "*" + } + ] + } + base_size = len(json.dumps(base_policy, separators=(',', ':'))) + + for permission in permissions: + # Calculate size if we add this permission + test_chunk = current_chunk + [permission] + test_policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": test_chunk, + "Resource": "*" + } + ] + } + test_size = len(json.dumps(test_policy, separators=(',', ':'))) + + # If adding this permission would exceed the limit, start a new chunk + if test_size > MAX_POLICY_SIZE and current_chunk: + chunks.append(current_chunk) + current_chunk = [permission] + current_size = len(json.dumps({ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [permission], + "Resource": "*" + } + ] + }, separators=(',', ':'))) + else: + current_chunk.append(permission) + current_size = test_size + + # Add the last chunk if it has permissions + if current_chunk: + chunks.append(current_chunk) + + return chunks + + def fetch_permissions_from_datadog(): + """Fetch permissions from Datadog API.""" + api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions" + headers = { + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = Request(api_url, headers=headers) + request.get_method = lambda: "GET" + + response = urllib.request.urlopen(request) + json_response = json.loads(response.read()) + if response.getcode() != 200: + error_message = json_response.get('errors', ['Unknown error'])[0] + raise DatadogAPIError(f"Datadog API error: {error_message}") + + return json_response["data"]["attributes"]["permissions"] + + def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name, max_policies=20): + """Clean up existing policies with both old and new naming patterns""" + + # Clean up new dynamic policies (datadog-aws-integration-iam-permissions-part{N}) + for i in range(max_policies): + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_arn = get_policy_arn(account_id, policy_name) + try: + iam_client.detach_role_policy( + RoleName=role_name, + PolicyArn=policy_arn + ) + except iam_client.exceptions.NoSuchEntityException: + # Policy to detach doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error detaching policy {policy_name}: {str(e)}") + + try: + iam_client.delete_policy( + PolicyArn=policy_arn + ) + except iam_client.exceptions.NoSuchEntityException: + # Policy to delete doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + + # Clean up old hardcoded managed policies ({IAMRoleName}-ManagedPolicy-{N}) + # Extract role name from the base_policy_name for old policy cleanup + role_name_from_hash = role_name # We have the role name directly + for i in range(1, 5): # Old template had ManagedPolicy-1 through ManagedPolicy-4 + old_policy_name = f"{role_name_from_hash}-ManagedPolicy-{i}" + old_policy_arn = get_policy_arn(account_id, old_policy_name) + try: + iam_client.detach_role_policy( + RoleName=role_name, + PolicyArn=old_policy_arn + ) + LOGGER.info(f"Detached old policy: {old_policy_name}") + except iam_client.exceptions.NoSuchEntityException: + # Policy to detach doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error detaching old policy {old_policy_name}: {str(e)}") + + try: + iam_client.delete_policy( + PolicyArn=old_policy_arn + ) + LOGGER.info(f"Deleted old policy: {old_policy_name}") + except iam_client.exceptions.NoSuchEntityException: + # Policy to delete doesn't exist + pass + except Exception as e: + LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") + + def handle_delete(event, context, role_name, account_id, base_policy_name): + """Handle stack deletion.""" + iam_client = boto3.client('iam') + try: + cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error deleting policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + + def handle_create_update(event, context, role_name, account_id, base_policy_name): + """Handle stack creation or update.""" + try: + # Fetch and smart chunk permissions based on character limits + permissions = fetch_permissions_from_datadog() + permission_chunks = create_smart_chunks(permissions) + + LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # Clean up existing policies + iam_client = boto3.client('iam') + cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + + # Create and attach new policies + for i, chunk in enumerate(permission_chunks): + # Create policy + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": chunk, + "Resource": "*" + } + ] + } + + # Verify policy size before creating + policy_json = json.dumps(policy_document, separators=(',', ':')) + policy_size = len(policy_json) + LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + + if policy_size > MAX_POLICY_SIZE: + LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + + policy = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=policy_json + ) + + # Attach policy to role + iam_client.attach_role_policy( + RoleName=role_name, + PolicyArn=policy['Policy']['Arn'] + ) + + # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled + #if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": + # iam_client.attach_role_policy( + # RoleName=role_name, + # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) + # ) + # + + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error creating/attaching policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + + def handler(event, context): + LOGGER.info("Event received: %s", json.dumps(event)) + + role_name = event['ResourceProperties']['DatadogIntegrationRole'] + account_id = event['ResourceProperties']['AccountId'] + + if event['RequestType'] == 'Delete': + handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + else: + handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) + DatadogAttachIntegrationPermissionsFunctionTrigger: + Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger Condition: ShouldInstallSecurityAuditPolicy Properties: - ManagedPolicyName: !Sub - - "${IAMRoleName}-ManagedPolicy-4" - - { IAMRoleName: !Ref IAMRoleName } - Roles: - - !Ref DatadogIntegrationRole - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Resource: "*" - Action: - - "ses:ListCustomVerificationEmailTemplates" - - "ses:ListIngressPoints" - - "ses:ListMultiRegionEndpoints" - - "ses:ListRelays" - - "ses:ListRuleSets" - - "ses:ListTemplates" - - "ses:ListTrafficPolicies" - - "signer:GetSigningProfile" - - "signer:ListSigningProfiles" - - "sms-voice:DescribeConfigurationSets" - - "sms-voice:DescribeOptOutLists" - - "sms-voice:DescribePhoneNumbers" - - "sms-voice:DescribePools" - - "sms-voice:DescribeProtectConfigurations" - - "sms-voice:DescribeRegistrationAttachments" - - "sms-voice:DescribeRegistrations" - - "sms-voice:DescribeSenderIds" - - "sms-voice:DescribeVerifiedDestinationNumbers" - - "snowball:DescribeCluster" - - "snowball:DescribeJob" - - "sns:ListEndpointsByPlatformApplication" - - "sns:ListPlatformApplications" - - "social-messaging:GetLinkedWhatsAppBusinessAccount" - - "social-messaging:ListLinkedWhatsAppBusinessAccounts" - - "sqs:GetQueueUrl" - - "ssm-incidents:GetIncidentRecord" - - "ssm-incidents:GetReplicationSet" - - "ssm-incidents:GetResponsePlan" - - "ssm-incidents:ListIncidentRecords" - - "ssm-incidents:ListReplicationSets" - - "ssm-incidents:ListResponsePlans" - - "ssm:GetMaintenanceWindow" - - "ssm:GetOpsItem" - - "ssm:GetPatchBaseline" - - "states:ListActivities" - - "states:ListExecutions" - - "states:ListMapRuns" - - "states:ListStateMachineAliases" - - "storagegateway:DescribeFileSystemAssociations" - - "storagegateway:DescribeSMBFileShares" - - "textract:GetAdapter" - - "textract:GetAdapterVersion" - - "textract:ListAdapterVersions" - - "textract:ListAdapters" - - "timestream:ListScheduledQueries" - - "timestream:ListTables" - - "transcribe:GetCallAnalyticsJob" - - "transcribe:GetMedicalScribeJob" - - "transcribe:GetMedicalTranscriptionJob" - - "transcribe:GetTranscriptionJob" - - "transcribe:ListMedicalScribeJobs" - - "translate:GetParallelData" - - "translate:GetTerminology" - - "verifiedpermissions:GetPolicyStore" - - "verifiedpermissions:ListIdentitySources" - - "verifiedpermissions:ListPolicies" - - "verifiedpermissions:ListPolicyStores" - - "verifiedpermissions:ListPolicyTemplates" - - "vpc-lattice:GetListener" - - "vpc-lattice:GetResourceConfiguration" - - "vpc-lattice:GetResourceGateway" - - "vpc-lattice:GetRule" - - "vpc-lattice:GetService" - - "vpc-lattice:GetServiceNetwork" - - "vpc-lattice:GetTargetGroup" - - "vpc-lattice:ListAccessLogSubscriptions" - - "vpc-lattice:ListListeners" - - "vpc-lattice:ListResourceConfigurations" - - "vpc-lattice:ListResourceEndpointAssociations" - - "vpc-lattice:ListResourceGateways" - - "vpc-lattice:ListRules" - - "vpc-lattice:ListServiceNetworkResourceAssociations" - - "vpc-lattice:ListServiceNetworkServiceAssociations" - - "vpc-lattice:ListServiceNetworkVpcAssociations" - - "vpc-lattice:ListServiceNetworks" - - "vpc-lattice:ListServices" - - "vpc-lattice:ListTargetGroups" - - "waf-regional:GetRule" - - "waf-regional:GetRuleGroup" - - "waf-regional:ListRuleGroups" - - "waf-regional:ListRules" - - "waf:GetRule" - - "waf:GetRuleGroup" - - "waf:ListRuleGroups" - - "waf:ListRules" - - "wafv2:GetIPSet" - - "wafv2:GetRegexPatternSet" - - "wafv2:GetRuleGroup" - - "workmail:DescribeOrganization" - - "workmail:ListOrganizations" - - "workspaces-web:GetBrowserSettings" - - "workspaces-web:GetDataProtectionSettings" - - "workspaces-web:GetIdentityProvider" - - "workspaces-web:GetIpAccessSettings" - - "workspaces-web:GetNetworkSettings" - - "workspaces-web:GetTrustStore" - - "workspaces-web:GetUserAccessLoggingSettings" - - "workspaces-web:GetUserSettings" - - "workspaces-web:ListBrowserSettings" - - "workspaces-web:ListDataProtectionSettings" - - "workspaces-web:ListIdentityProviders" - - "workspaces-web:ListIpAccessSettings" - - "workspaces-web:ListNetworkSettings" - - "workspaces-web:ListPortals" - - "workspaces-web:ListTrustStores" - - "workspaces-web:ListUserAccessLoggingSettings" - - "workspaces-web:ListUserSettings" + ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn + DatadogIntegrationRole: !Ref IAMRoleName + AccountId: !Ref AWS::AccountId + Partition: !Ref AWS::Partition + ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: ParameterGroups: From 09977f6afe6388d1233378042bce40abdadc9f39 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 16:47:46 -0400 Subject: [PATCH 02/32] handle delete conflict exception --- aws_quickstart/datadog_integration_role.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 78f60e76..e294f7e1 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -244,7 +244,6 @@ Resources: """Create chunks based on character limit rather than permission count.""" chunks = [] current_chunk = [] - current_size = 0 # Base policy structure size (without permissions) base_policy = { @@ -337,7 +336,7 @@ Resources: iam_client.delete_policy( PolicyArn=policy_arn ) - except iam_client.exceptions.NoSuchEntityException: + except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): # Policy to delete doesn't exist pass except Exception as e: @@ -366,7 +365,7 @@ Resources: PolicyArn=old_policy_arn ) LOGGER.info(f"Deleted old policy: {old_policy_name}") - except iam_client.exceptions.NoSuchEntityException: + except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): # Policy to delete doesn't exist pass except Exception as e: @@ -430,13 +429,14 @@ Resources: PolicyArn=policy['Policy']['Arn'] ) - # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled - #if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": - # iam_client.attach_role_policy( - # RoleName=role_name, - # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) - # ) - # + # probably not needed######################################################## + # # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled + # if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": + # iam_client.attach_role_policy( + # RoleName=role_name, + # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) + # ) + ######################################################### cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: From fd5f548cb047ff72f3a39526d11467536691e6f6 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 16:48:33 -0400 Subject: [PATCH 03/32] remove standard inline policy --- aws_quickstart/datadog_integration_role.yaml | 112 ------------------- 1 file changed, 112 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index e294f7e1..b372fe2e 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -61,118 +61,6 @@ Resources: [!Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit"], !Ref AWS::NoValue, ] - Policies: - - PolicyName: DatadogAWSIntegrationPolicy - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Resource: "*" - Action: - - "account:GetAccountInformation" - - "airflow:GetEnvironment" - - "airflow:ListEnvironments" - - "apigateway:GET" - - "appsync:ListGraphqlApis" - - "autoscaling:Describe*" - - "backup:List*" - - "batch:DescribeJobDefinitions" - - "bcm-data-exports:GetExport" - - "bcm-data-exports:ListExports" - - "budgets:ViewBudget" - - "cloudfront:GetDistributionConfig" - - "cloudfront:ListDistributions" - - "cloudtrail:DescribeTrails" - - "cloudtrail:GetTrail" - - "cloudtrail:GetTrailStatus" - - "cloudtrail:ListTrails" - - "cloudtrail:LookupEvents" - - "cloudwatch:Describe*" - - "cloudwatch:Get*" - - "cloudwatch:List*" - - "codebuild:BatchGetProjects" - - "codebuild:ListProjects" - - "codedeploy:BatchGet*" - - "codedeploy:List*" - - "cur:DescribeReportDefinitions" - - "directconnect:Describe*" - - "dms:DescribeReplicationInstances" - - "dynamodb:Describe*" - - "dynamodb:List*" - - "ec2:Describe*" - - "ecs:Describe*" - - "ecs:List*" - - "eks:DescribeCluster" - - "eks:ListClusters" - - "elasticache:Describe*" - - "elasticache:List*" - - "elasticfilesystem:DescribeAccessPoints" - - "elasticfilesystem:DescribeFileSystems" - - "elasticfilesystem:DescribeTags" - - "elasticloadbalancing:Describe*" - - "elasticmapreduce:Describe*" - - "elasticmapreduce:List*" - - "es:DescribeElasticsearchDomains" - - "es:ListDomainNames" - - "es:ListTags" - - "events:CreateEventBus" - - "fsx:DescribeFileSystems" - - "fsx:ListTagsForResource" - - "health:DescribeAffectedEntities" - - "health:DescribeEventDetails" - - "health:DescribeEvents" - - "iam:ListAccountAliases" - - "kinesis:Describe*" - - "kinesis:List*" - - "lambda:List*" - - "logs:DeleteSubscriptionFilter" - - "logs:DescribeDeliveries" - - "logs:DescribeDeliverySources" - - "logs:DescribeLogGroups" - - "logs:DescribeLogStreams" - - "logs:DescribeSubscriptionFilters" - - "logs:FilterLogEvents" - - "logs:GetDeliveryDestination" - - "logs:PutSubscriptionFilter" - - "logs:TestMetricFilter" - - "network-firewall:DescribeLoggingConfiguration" - - "network-firewall:ListFirewalls" - - "oam:ListAttachedLinks" - - "oam:ListSinks" - - "organizations:Describe*" - - "organizations:List*" - - "rds:Describe*" - - "rds:List*" - - "redshift-serverless:ListNamespaces" - - "redshift:DescribeClusters" - - "redshift:DescribeLoggingStatus" - - "route53:List*" - - "route53resolver:ListResolverQueryLogConfigs" - - "s3:GetBucketLocation" - - "s3:GetBucketLogging" - - "s3:GetBucketNotification" - - "s3:GetBucketTagging" - - "s3:ListAllMyBuckets" - - "s3:PutBucketNotification" - - "ses:Get*" - - "ses:List*" - - "sns:GetSubscriptionAttributes" - - "sns:List*" - - "sns:Publish" - - "sqs:ListQueues" - - "ssm:GetServiceSetting" - - "ssm:ListCommands" - - "states:DescribeStateMachine" - - "states:ListStateMachines" - - "support:DescribeTrustedAdvisor*" - - "support:RefreshTrustedAdvisorCheck" - - "tag:GetResources" - - "tag:GetTagKeys" - - "tag:GetTagValues" - - "timestream:DescribeEndpoints" - - "wafv2:ListLoggingConfigurations" - - "xray:BatchGetTraces" - - "xray:GetTraceSummaries" # Lambda function to dynamically attach permissions from DataDog API DatadogAttachIntegrationPermissionsLambdaExecutionRole: Type: AWS::IAM::Role From 5290d7e9f45ecef2bd9222a169e1978ef662d9a0 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 16:59:13 -0400 Subject: [PATCH 04/32] add managed policy prefix to allowed resources in lambda execution role --- aws_quickstart/datadog_integration_role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index b372fe2e..0c57706a 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -92,6 +92,7 @@ Resources: Resource: - !Sub arn:aws:iam::${AWS::AccountId}:role/${IAMRoleName} - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-iam-permissions-* + - !Sub arn:aws:iam::${AWS::AccountId}:policy/${IAMRoleName}-ManagedPolicy-* - !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" DatadogAttachIntegrationPermissionsFunction: Type: AWS::Lambda::Function From e6a3d12380219c0bbc62fefa319a43446c97e7fd Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 17:39:02 -0400 Subject: [PATCH 05/32] use inline policy for standard permissions --- .../attach_integration_permissions.py | 180 +++++++++++----- aws_quickstart/datadog_integration_role.yaml | 200 ++++++++++++------ 2 files changed, 266 insertions(+), 114 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 1ce65317..855c5d72 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -11,6 +11,8 @@ API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" +STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" +RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" class DatadogAPIError(Exception): pass @@ -76,9 +78,9 @@ def create_smart_chunks(permissions): return chunks -def fetch_permissions_from_datadog(): +def fetch_permissions_from_datadog(api_url): """Fetch permissions from Datadog API.""" - api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" + # api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" headers = { "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } @@ -150,6 +152,80 @@ def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_nam except Exception as e: LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") +def attach_standard_permissions(iam_client, role_name): + # Fetch permissions + permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) + # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_name = "DatadogAWSIntegrationPolicy" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": permissions, + "Resource": "*" + } + ] + } + + # policy = iam_client.create_policy( + # PolicyName=policy_name, + # PolicyDocument=json.dumps(policy_document, separators=(',', ':')) + # ) + # Attach policy to role + iam_client.put_role_policy( + RoleName=role_name, + PolicyName=policy_name, + PolicyDOcument=json.dumps(policy_document, separators=(',', ':')) + ) + + +def attach_resource_collection_permissions(iam_client): + # Fetch and smart chunk permissions based on character limits + permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + permission_chunks = create_smart_chunks(permissions) + + LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # Clean up existing policies + iam_client = boto3.client('iam') + # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + + # Create and attach new policies + for i, chunk in enumerate(permission_chunks): + # Create policy + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": chunk, + "Resource": "*" + } + ] + } + + # Verify policy size before creating + policy_json = json.dumps(policy_document, separators=(',', ':')) + policy_size = len(policy_json) + LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + + if policy_size > MAX_POLICY_SIZE: + LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + + policy = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=policy_json + ) + + # Attach policy to role + iam_client.attach_role_policy( + RoleName=role_name, + PolicyArn=policy['Policy']['Arn'] + ) + def handle_delete(event, context, role_name, account_id, base_policy_name): """Handle stack deletion.""" iam_client = boto3.client('iam') @@ -163,64 +239,64 @@ def handle_delete(event, context, role_name, account_id, base_policy_name): def handle_create_update(event, context, role_name, account_id, base_policy_name): """Handle stack creation or update.""" try: - # Fetch and smart chunk permissions based on character limits - permissions = fetch_permissions_from_datadog() - permission_chunks = create_smart_chunks(permissions) - - LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - - # Clean up existing policies iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + attach_standard_permissions(iam_client) + attach_resource_collection_permissions(iam_client) + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error creating/attaching policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + # try: + # # Fetch and smart chunk permissions based on character limits + # permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + # permission_chunks = create_smart_chunks(permissions) + + # LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # # Clean up existing policies + # iam_client = boto3.client('iam') + # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - # Create and attach new policies - for i, chunk in enumerate(permission_chunks): - # Create policy - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_document = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": chunk, - "Resource": "*" - } - ] - } + # # Create and attach new policies + # for i, chunk in enumerate(permission_chunks): + # # Create policy + # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + # policy_document = { + # "Version": "2012-10-17", + # "Statement": [ + # { + # "Effect": "Allow", + # "Action": chunk, + # "Resource": "*" + # } + # ] + # } - # Verify policy size before creating - policy_json = json.dumps(policy_document, separators=(',', ':')) - policy_size = len(policy_json) - LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + # # Verify policy size before creating + # policy_json = json.dumps(policy_document, separators=(',', ':')) + # policy_size = len(policy_json) + # LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - if policy_size > MAX_POLICY_SIZE: - LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + # if policy_size > MAX_POLICY_SIZE: + # LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + # raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - policy = iam_client.create_policy( - PolicyName=policy_name, - PolicyDocument=policy_json - ) + # policy = iam_client.create_policy( + # PolicyName=policy_name, + # PolicyDocument=policy_json + # ) - # Attach policy to role - iam_client.attach_role_policy( - RoleName=role_name, - PolicyArn=policy['Policy']['Arn'] - ) - - # probably not needed######################################################## - # # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled - # if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": - # iam_client.attach_role_policy( - # RoleName=role_name, - # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) - # ) - ######################################################### + # # Attach policy to role + # iam_client.attach_role_policy( + # RoleName=role_name, + # PolicyArn=policy['Policy']['Arn'] + # ) - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) - except Exception as e: - LOGGER.error(f"Error creating/attaching policy: {str(e)}") - cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + # cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + # except Exception as e: + # LOGGER.error(f"Error creating/attaching policy: {str(e)}") + # cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) def handler(event, context): LOGGER.info("Event received: %s", json.dumps(event)) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 0c57706a..6d48e41b 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -121,6 +121,8 @@ Resources: API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" + STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" + RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" class DatadogAPIError(Exception): pass @@ -186,9 +188,9 @@ Resources: return chunks - def fetch_permissions_from_datadog(): + def fetch_permissions_from_datadog(api_url): """Fetch permissions from Datadog API.""" - api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions" + # api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" headers = { "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } @@ -260,6 +262,80 @@ Resources: except Exception as e: LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") + def attach_standard_permissions(iam_client, role_name): + # Fetch permissions + permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) + # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_name = "DatadogAWSIntegrationPolicy" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": permissions, + "Resource": "*" + } + ] + } + + # policy = iam_client.create_policy( + # PolicyName=policy_name, + # PolicyDocument=json.dumps(policy_document, separators=(',', ':')) + # ) + # Attach policy to role + iam_client.put_role_policy( + RoleName=role_name, + PolicyName=policy_name, + PolicyDOcument=json.dumps(policy_document, separators=(',', ':')) + ) + + + def attach_resource_collection_permissions(iam_client): + # Fetch and smart chunk permissions based on character limits + permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + permission_chunks = create_smart_chunks(permissions) + + LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # Clean up existing policies + iam_client = boto3.client('iam') + # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + + # Create and attach new policies + for i, chunk in enumerate(permission_chunks): + # Create policy + policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_document = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": chunk, + "Resource": "*" + } + ] + } + + # Verify policy size before creating + policy_json = json.dumps(policy_document, separators=(',', ':')) + policy_size = len(policy_json) + LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + + if policy_size > MAX_POLICY_SIZE: + LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + + policy = iam_client.create_policy( + PolicyName=policy_name, + PolicyDocument=policy_json + ) + + # Attach policy to role + iam_client.attach_role_policy( + RoleName=role_name, + PolicyArn=policy['Policy']['Arn'] + ) + def handle_delete(event, context, role_name, account_id, base_policy_name): """Handle stack deletion.""" iam_client = boto3.client('iam') @@ -273,75 +349,75 @@ Resources: def handle_create_update(event, context, role_name, account_id, base_policy_name): """Handle stack creation or update.""" try: - # Fetch and smart chunk permissions based on character limits - permissions = fetch_permissions_from_datadog() - permission_chunks = create_smart_chunks(permissions) - - LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - - # Clean up existing policies iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + attach_standard_permissions(iam_client) + attach_resource_collection_permissions(iam_client) + cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + except Exception as e: + LOGGER.error(f"Error creating/attaching policy: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + # try: + # # Fetch and smart chunk permissions based on character limits + # permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + # permission_chunks = create_smart_chunks(permissions) + + # LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") + + # # Clean up existing policies + # iam_client = boto3.client('iam') + # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - # Create and attach new policies - for i, chunk in enumerate(permission_chunks): - # Create policy - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_document = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": chunk, - "Resource": "*" - } - ] - } + # # Create and attach new policies + # for i, chunk in enumerate(permission_chunks): + # # Create policy + # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + # policy_document = { + # "Version": "2012-10-17", + # "Statement": [ + # { + # "Effect": "Allow", + # "Action": chunk, + # "Resource": "*" + # } + # ] + # } - # Verify policy size before creating - policy_json = json.dumps(policy_document, separators=(',', ':')) - policy_size = len(policy_json) - LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") + # # Verify policy size before creating + # policy_json = json.dumps(policy_document, separators=(',', ':')) + # policy_size = len(policy_json) + # LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - if policy_size > MAX_POLICY_SIZE: - LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") + # if policy_size > MAX_POLICY_SIZE: + # LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") + # raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - policy = iam_client.create_policy( - PolicyName=policy_name, - PolicyDocument=policy_json - ) + # policy = iam_client.create_policy( + # PolicyName=policy_name, + # PolicyDocument=policy_json + # ) - # Attach policy to role - iam_client.attach_role_policy( - RoleName=role_name, - PolicyArn=policy['Policy']['Arn'] - ) - - # probably not needed######################################################## - # # Attach the SecurityAudit policy if ResourceCollectionPermissions is enabled - # if event["ResourceProperties"].get("ResourceCollectionPermissions", "false") == "true": - # iam_client.attach_role_policy( - # RoleName=role_name, - # PolicyArn="arn:{partition}:iam::aws:policy/SecurityAudit".format(partition=event["ResourceProperties"]["Partition"]) - # ) - ######################################################### + # # Attach policy to role + # iam_client.attach_role_policy( + # RoleName=role_name, + # PolicyArn=policy['Policy']['Arn'] + # ) - cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) - except Exception as e: - LOGGER.error(f"Error creating/attaching policy: {str(e)}") - cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) + # cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) + # except Exception as e: + # LOGGER.error(f"Error creating/attaching policy: {str(e)}") + # cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) +def handler(event, context): + LOGGER.info("Event received: %s", json.dumps(event)) + + role_name = event['ResourceProperties']['DatadogIntegrationRole'] + account_id = event['ResourceProperties']['AccountId'] + + if event['RequestType'] == 'Delete': + handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + else: + handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) - def handler(event, context): - LOGGER.info("Event received: %s", json.dumps(event)) - - role_name = event['ResourceProperties']['DatadogIntegrationRole'] - account_id = event['ResourceProperties']['AccountId'] - - if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) - else: - handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger Condition: ShouldInstallSecurityAuditPolicy From d7bef901f99f27b6f7646cbf36a66c4708077d37 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 17:39:50 -0400 Subject: [PATCH 06/32] use inline policy for standard permissions --- aws_quickstart/datadog_integration_role.yaml | 38 ++++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 6d48e41b..0387a13c 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -407,26 +407,26 @@ Resources: # except Exception as e: # LOGGER.error(f"Error creating/attaching policy: {str(e)}") # cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) -def handler(event, context): - LOGGER.info("Event received: %s", json.dumps(event)) - - role_name = event['ResourceProperties']['DatadogIntegrationRole'] - account_id = event['ResourceProperties']['AccountId'] - - if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) - else: - handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) + def handler(event, context): + LOGGER.info("Event received: %s", json.dumps(event)) + + role_name = event['ResourceProperties']['DatadogIntegrationRole'] + account_id = event['ResourceProperties']['AccountId'] + + if event['RequestType'] == 'Delete': + handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + else: + handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) - DatadogAttachIntegrationPermissionsFunctionTrigger: - Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger - Condition: ShouldInstallSecurityAuditPolicy - Properties: - ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn - DatadogIntegrationRole: !Ref IAMRoleName - AccountId: !Ref AWS::AccountId - Partition: !Ref AWS::Partition - ResourceCollectionPermissions: !Ref ResourceCollectionPermissions + DatadogAttachIntegrationPermissionsFunctionTrigger: + Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger + Condition: ShouldInstallSecurityAuditPolicy + Properties: + ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn + DatadogIntegrationRole: !Ref IAMRoleName + AccountId: !Ref AWS::AccountId + Partition: !Ref AWS::Partition + ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: ParameterGroups: From a0da5e9b4ab0a923acded631ebcfbee992b2d619 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 17:52:01 -0400 Subject: [PATCH 07/32] readd lambda trigger --- aws_quickstart/datadog_integration_role.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 0387a13c..4c04600a 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -417,16 +417,15 @@ Resources: handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) else: handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) - - DatadogAttachIntegrationPermissionsFunctionTrigger: - Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger - Condition: ShouldInstallSecurityAuditPolicy - Properties: - ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn - DatadogIntegrationRole: !Ref IAMRoleName - AccountId: !Ref AWS::AccountId - Partition: !Ref AWS::Partition - ResourceCollectionPermissions: !Ref ResourceCollectionPermissions + DatadogAttachIntegrationPermissionsFunctionTrigger: + Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger + Condition: ShouldInstallSecurityAuditPolicy + Properties: + ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn + DatadogIntegrationRole: !Ref IAMRoleName + AccountId: !Ref AWS::AccountId + Partition: !Ref AWS::Partition + ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: ParameterGroups: From f4209183ee4fe7ee36247a136b80842839a388e5 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 17:56:11 -0400 Subject: [PATCH 08/32] add missing param --- aws_quickstart/attach_integration_permissions.py | 2 +- aws_quickstart/datadog_integration_role.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 855c5d72..4a54cb90 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -241,7 +241,7 @@ def handle_create_update(event, context, role_name, account_id, base_policy_name try: iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - attach_standard_permissions(iam_client) + attach_standard_permissions(iam_client, role_name) attach_resource_collection_permissions(iam_client) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 4c04600a..053fc698 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -351,7 +351,7 @@ Resources: try: iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - attach_standard_permissions(iam_client) + attach_standard_permissions(iam_client, role_name) attach_resource_collection_permissions(iam_client) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: From f0d6ea9195ab71acc652b70024932d3b4d1b27d0 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 18:00:51 -0400 Subject: [PATCH 09/32] fix typo --- aws_quickstart/datadog_integration_role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 053fc698..049f81c9 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -286,7 +286,7 @@ Resources: iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, - PolicyDOcument=json.dumps(policy_document, separators=(',', ':')) + PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) From d4bdd7ff200d6a46374fa825a81eaa1df603b599 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 18:21:20 -0400 Subject: [PATCH 10/32] add PutRolePolicy permission to lambda execution role --- aws_quickstart/datadog_integration_role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 049f81c9..51aee75b 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -89,6 +89,7 @@ Resources: - iam:DeletePolicy - iam:AttachRolePolicy - iam:DetachRolePolicy + - iam:PutRolePolicy Resource: - !Sub arn:aws:iam::${AWS::AccountId}:role/${IAMRoleName} - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-iam-permissions-* From 7768d8e7c348409f124909a8510af9f01d5bdb9b Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Wed, 22 Oct 2025 18:38:21 -0400 Subject: [PATCH 11/32] fix typo --- aws_quickstart/attach_integration_permissions.py | 6 +++--- aws_quickstart/datadog_integration_role.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 4a54cb90..3305e54b 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -176,11 +176,11 @@ def attach_standard_permissions(iam_client, role_name): iam_client.put_role_policy( RoleName=role_name, PolicyName=policy_name, - PolicyDOcument=json.dumps(policy_document, separators=(',', ':')) + PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) -def attach_resource_collection_permissions(iam_client): +def attach_resource_collection_permissions(iam_client, role_name): # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) permission_chunks = create_smart_chunks(permissions) @@ -242,7 +242,7 @@ def handle_create_update(event, context, role_name, account_id, base_policy_name iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) attach_standard_permissions(iam_client, role_name) - attach_resource_collection_permissions(iam_client) + attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 51aee75b..8526e038 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -64,7 +64,7 @@ Resources: # Lambda function to dynamically attach permissions from DataDog API DatadogAttachIntegrationPermissionsLambdaExecutionRole: Type: AWS::IAM::Role - Condition: ShouldInstallSecurityAuditPolicy + # Condition: ShouldInstallSecurityAuditPolicy Properties: AssumeRolePolicyDocument: Version: '2012-10-17' @@ -291,7 +291,7 @@ Resources: ) - def attach_resource_collection_permissions(iam_client): + def attach_resource_collection_permissions(iam_client, role_name): # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) permission_chunks = create_smart_chunks(permissions) @@ -353,7 +353,7 @@ Resources: iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) attach_standard_permissions(iam_client, role_name) - attach_resource_collection_permissions(iam_client) + attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") From 4e818fe87083099a7f19c1bb9afe6e8f0b85c2d1 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:12:31 -0400 Subject: [PATCH 12/32] cleanup lambda code --- .../attach_integration_permissions.py | 172 +++--------------- aws_quickstart/datadog_integration_role.yaml | 172 +++--------------- 2 files changed, 47 insertions(+), 297 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 3305e54b..36fb663c 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -1,6 +1,5 @@ import json import logging -import hashlib from urllib.request import Request import urllib import cfnresponse @@ -10,67 +9,40 @@ LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document -BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" +POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" +BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" class DatadogAPIError(Exception): pass -def get_policy_arn(account_id, policy_name): - """Generate a policy ARN.""" - return f"arn:aws:iam::{account_id}:policy/{policy_name}" - -def create_smart_chunks(permissions): - """Create chunks based on character limit rather than permission count.""" +def create_chunks(permissions): + """Create chunks of permissions based on character limit""" chunks = [] current_chunk = [] - # Base policy structure size (without permissions) - base_policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [], - "Resource": "*" - } - ] - } - base_size = len(json.dumps(base_policy, separators=(',', ':'))) - for permission in permissions: # Calculate size if we add this permission - test_chunk = current_chunk + [permission] - test_policy = { + next_chunk = current_chunk + [permission] + policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", - "Action": test_chunk, + "Action": next_chunk, "Resource": "*" } ] } - test_size = len(json.dumps(test_policy, separators=(',', ':'))) + policy_size = len(json.dumps(policy, separators=(',', ':'))) # If adding this permission would exceed the limit, start a new chunk - if test_size > MAX_POLICY_SIZE and current_chunk: + if policy_size > MAX_POLICY_SIZE and current_chunk: chunks.append(current_chunk) current_chunk = [permission] - current_size = len(json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [permission], - "Resource": "*" - } - ] - }, separators=(',', ':'))) else: current_chunk.append(permission) - current_size = test_size # Add the last chunk if it has permissions if current_chunk: @@ -79,8 +51,7 @@ def create_smart_chunks(permissions): return chunks def fetch_permissions_from_datadog(api_url): - """Fetch permissions from Datadog API.""" - # api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" + """Fetch permissions from Datadog API""" headers = { "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } @@ -95,13 +66,11 @@ def fetch_permissions_from_datadog(api_url): return json_response["data"]["attributes"]["permissions"] -def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name, max_policies=20): - """Clean up existing policies with both old and new naming patterns""" - - # Clean up new dynamic policies (datadog-aws-integration-iam-permissions-part{N}) +def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): + """Clean up existing policies""" for i in range(max_policies): - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_arn = get_policy_arn(account_id, policy_name) + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( RoleName=role_name, @@ -123,40 +92,8 @@ def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_nam except Exception as e: LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") - # Clean up old hardcoded managed policies ({IAMRoleName}-ManagedPolicy-{N}) - # Extract role name from the base_policy_name for old policy cleanup - role_name_from_hash = role_name # We have the role name directly - for i in range(1, 5): # Old template had ManagedPolicy-1 through ManagedPolicy-4 - old_policy_name = f"{role_name_from_hash}-ManagedPolicy-{i}" - old_policy_arn = get_policy_arn(account_id, old_policy_name) - try: - iam_client.detach_role_policy( - RoleName=role_name, - PolicyArn=old_policy_arn - ) - LOGGER.info(f"Detached old policy: {old_policy_name}") - except iam_client.exceptions.NoSuchEntityException: - # Policy to detach doesn't exist - pass - except Exception as e: - LOGGER.error(f"Error detaching old policy {old_policy_name}: {str(e)}") - - try: - iam_client.delete_policy( - PolicyArn=old_policy_arn - ) - LOGGER.info(f"Deleted old policy: {old_policy_name}") - except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): - # Policy to delete doesn't exist - pass - except Exception as e: - LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") - def attach_standard_permissions(iam_client, role_name): - # Fetch permissions permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) - # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_name = "DatadogAWSIntegrationPolicy" policy_document = { "Version": "2012-10-17", "Statement": [ @@ -168,14 +105,9 @@ def attach_standard_permissions(iam_client, role_name): ] } - # policy = iam_client.create_policy( - # PolicyName=policy_name, - # PolicyDocument=json.dumps(policy_document, separators=(',', ':')) - # ) - # Attach policy to role iam_client.put_role_policy( RoleName=role_name, - PolicyName=policy_name, + PolicyName=POLICY_NAME_STANDARD, PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) @@ -183,18 +115,14 @@ def attach_standard_permissions(iam_client, role_name): def attach_resource_collection_permissions(iam_client, role_name): # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - permission_chunks = create_smart_chunks(permissions) + permission_chunks = create_chunks(permissions) LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - # Clean up existing policies - iam_client = boto3.client('iam') - # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" policy_document = { "Version": "2012-10-17", "Statement": [ @@ -211,10 +139,6 @@ def attach_resource_collection_permissions(iam_client, role_name): policy_size = len(policy_json) LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - if policy_size > MAX_POLICY_SIZE: - LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - policy = iam_client.create_policy( PolicyName=policy_name, PolicyDocument=policy_json @@ -226,77 +150,27 @@ def attach_resource_collection_permissions(iam_client, role_name): PolicyArn=policy['Policy']['Arn'] ) -def handle_delete(event, context, role_name, account_id, base_policy_name): +def handle_delete(event, context, role_name, account_id): """Handle stack deletion.""" iam_client = boto3.client('iam') try: - cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cleanup_existing_policies(iam_client, role_name, account_id) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) -def handle_create_update(event, context, role_name, account_id, base_policy_name): +def handle_create_update(event, context, role_name, account_id): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') - cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cleanup_existing_policies(iam_client, role_name, account_id) attach_standard_permissions(iam_client, role_name) attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) - # try: - # # Fetch and smart chunk permissions based on character limits - # permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - # permission_chunks = create_smart_chunks(permissions) - - # LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - - # # Clean up existing policies - # iam_client = boto3.client('iam') - # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - - # # Create and attach new policies - # for i, chunk in enumerate(permission_chunks): - # # Create policy - # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - # policy_document = { - # "Version": "2012-10-17", - # "Statement": [ - # { - # "Effect": "Allow", - # "Action": chunk, - # "Resource": "*" - # } - # ] - # } - - # # Verify policy size before creating - # policy_json = json.dumps(policy_document, separators=(',', ':')) - # policy_size = len(policy_json) - # LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - - # if policy_size > MAX_POLICY_SIZE: - # LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - # raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - - # policy = iam_client.create_policy( - # PolicyName=policy_name, - # PolicyDocument=policy_json - # ) - - # # Attach policy to role - # iam_client.attach_role_policy( - # RoleName=role_name, - # PolicyArn=policy['Policy']['Arn'] - # ) - - # cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) - # except Exception as e: - # LOGGER.error(f"Error creating/attaching policy: {str(e)}") - # cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) def handler(event, context): LOGGER.info("Event received: %s", json.dumps(event)) @@ -305,6 +179,6 @@ def handler(event, context): account_id = event['ResourceProperties']['AccountId'] if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + handle_delete(event, context, role_name, account_id) else: - handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) + handle_create_update(event, context, role_name, account_id) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 8526e038..3a59a41f 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -111,7 +111,6 @@ Resources: ZipFile: | import json import logging - import hashlib from urllib.request import Request import urllib import cfnresponse @@ -121,67 +120,40 @@ Resources: LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document - BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" + POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" + BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" class DatadogAPIError(Exception): pass - def get_policy_arn(account_id, policy_name): - """Generate a policy ARN.""" - return f"arn:aws:iam::{account_id}:policy/{policy_name}" - - def create_smart_chunks(permissions): - """Create chunks based on character limit rather than permission count.""" + def create_chunks(permissions): + """Create chunks of permissions based on character limit""" chunks = [] current_chunk = [] - # Base policy structure size (without permissions) - base_policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [], - "Resource": "*" - } - ] - } - base_size = len(json.dumps(base_policy, separators=(',', ':'))) - for permission in permissions: # Calculate size if we add this permission - test_chunk = current_chunk + [permission] - test_policy = { + next_chunk = current_chunk + [permission] + policy = { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", - "Action": test_chunk, + "Action": next_chunk, "Resource": "*" } ] } - test_size = len(json.dumps(test_policy, separators=(',', ':'))) + policy_size = len(json.dumps(policy, separators=(',', ':'))) # If adding this permission would exceed the limit, start a new chunk - if test_size > MAX_POLICY_SIZE and current_chunk: + if policy_size > MAX_POLICY_SIZE and current_chunk: chunks.append(current_chunk) current_chunk = [permission] - current_size = len(json.dumps({ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [permission], - "Resource": "*" - } - ] - }, separators=(',', ':'))) else: current_chunk.append(permission) - current_size = test_size # Add the last chunk if it has permissions if current_chunk: @@ -190,8 +162,7 @@ Resources: return chunks def fetch_permissions_from_datadog(api_url): - """Fetch permissions from Datadog API.""" - # api_url = f"https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" + """Fetch permissions from Datadog API""" headers = { "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, } @@ -206,13 +177,11 @@ Resources: return json_response["data"]["attributes"]["permissions"] - def cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name, max_policies=20): - """Clean up existing policies with both old and new naming patterns""" - - # Clean up new dynamic policies (datadog-aws-integration-iam-permissions-part{N}) + def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): + """Clean up existing policies""" for i in range(max_policies): - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_arn = get_policy_arn(account_id, policy_name) + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( RoleName=role_name, @@ -234,40 +203,8 @@ Resources: except Exception as e: LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") - # Clean up old hardcoded managed policies ({IAMRoleName}-ManagedPolicy-{N}) - # Extract role name from the base_policy_name for old policy cleanup - role_name_from_hash = role_name # We have the role name directly - for i in range(1, 5): # Old template had ManagedPolicy-1 through ManagedPolicy-4 - old_policy_name = f"{role_name_from_hash}-ManagedPolicy-{i}" - old_policy_arn = get_policy_arn(account_id, old_policy_name) - try: - iam_client.detach_role_policy( - RoleName=role_name, - PolicyArn=old_policy_arn - ) - LOGGER.info(f"Detached old policy: {old_policy_name}") - except iam_client.exceptions.NoSuchEntityException: - # Policy to detach doesn't exist - pass - except Exception as e: - LOGGER.error(f"Error detaching old policy {old_policy_name}: {str(e)}") - - try: - iam_client.delete_policy( - PolicyArn=old_policy_arn - ) - LOGGER.info(f"Deleted old policy: {old_policy_name}") - except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): - # Policy to delete doesn't exist - pass - except Exception as e: - LOGGER.error(f"Error deleting old policy {old_policy_name}: {str(e)}") - def attach_standard_permissions(iam_client, role_name): - # Fetch permissions permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) - # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - policy_name = "DatadogAWSIntegrationPolicy" policy_document = { "Version": "2012-10-17", "Statement": [ @@ -279,14 +216,9 @@ Resources: ] } - # policy = iam_client.create_policy( - # PolicyName=policy_name, - # PolicyDocument=json.dumps(policy_document, separators=(',', ':')) - # ) - # Attach policy to role iam_client.put_role_policy( RoleName=role_name, - PolicyName=policy_name, + PolicyName=POLICY_NAME_STANDARD, PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) @@ -294,18 +226,14 @@ Resources: def attach_resource_collection_permissions(iam_client, role_name): # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - permission_chunks = create_smart_chunks(permissions) + permission_chunks = create_chunks(permissions) LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - # Clean up existing policies - iam_client = boto3.client('iam') - # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy - policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" policy_document = { "Version": "2012-10-17", "Statement": [ @@ -322,10 +250,6 @@ Resources: policy_size = len(policy_json) LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - if policy_size > MAX_POLICY_SIZE: - LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - policy = iam_client.create_policy( PolicyName=policy_name, PolicyDocument=policy_json @@ -337,77 +261,28 @@ Resources: PolicyArn=policy['Policy']['Arn'] ) - def handle_delete(event, context, role_name, account_id, base_policy_name): + def handle_delete(event, context, role_name, account_id): """Handle stack deletion.""" iam_client = boto3.client('iam') try: - cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cleanup_existing_policies(iam_client, role_name, account_id) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) - def handle_create_update(event, context, role_name, account_id, base_policy_name): + def handle_create_update(event, context, role_name, account_id): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') - cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) + cleanup_existing_policies(iam_client, role_name, account_id) attach_standard_permissions(iam_client, role_name) attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) - # try: - # # Fetch and smart chunk permissions based on character limits - # permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - # permission_chunks = create_smart_chunks(permissions) - - # LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - - # # Clean up existing policies - # iam_client = boto3.client('iam') - # cleanup_existing_policies(iam_client, role_name, account_id, base_policy_name) - # # Create and attach new policies - # for i, chunk in enumerate(permission_chunks): - # # Create policy - # policy_name = f"{BASE_POLICY_PREFIX}-part{i+1}" - # policy_document = { - # "Version": "2012-10-17", - # "Statement": [ - # { - # "Effect": "Allow", - # "Action": chunk, - # "Resource": "*" - # } - # ] - # } - - # # Verify policy size before creating - # policy_json = json.dumps(policy_document, separators=(',', ':')) - # policy_size = len(policy_json) - # LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - - # if policy_size > MAX_POLICY_SIZE: - # LOGGER.error(f"Policy {policy_name} exceeds size limit: {policy_size} > {MAX_POLICY_SIZE}") - # raise Exception(f"Policy size exceeds AWS limit: {policy_size} > {MAX_POLICY_SIZE}") - - # policy = iam_client.create_policy( - # PolicyName=policy_name, - # PolicyDocument=policy_json - # ) - - # # Attach policy to role - # iam_client.attach_role_policy( - # RoleName=role_name, - # PolicyArn=policy['Policy']['Arn'] - # ) - - # cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) - # except Exception as e: - # LOGGER.error(f"Error creating/attaching policy: {str(e)}") - # cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) def handler(event, context): LOGGER.info("Event received: %s", json.dumps(event)) @@ -415,9 +290,10 @@ Resources: account_id = event['ResourceProperties']['AccountId'] if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id, BASE_POLICY_PREFIX) + handle_delete(event, context, role_name, account_id) else: - handle_create_update(event, context, role_name, account_id, BASE_POLICY_PREFIX) + handle_create_update(event, context, role_name, account_id) + DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger Condition: ShouldInstallSecurityAuditPolicy From 40da360c3f19b15a2000d8ed932373c752183b15 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:24:21 -0400 Subject: [PATCH 13/32] fix typo --- aws_quickstart/attach_integration_permissions.py | 2 +- aws_quickstart/datadog_integration_role.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 36fb663c..b0b3bb2f 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -69,7 +69,7 @@ def fetch_permissions_from_datadog(api_url): def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): """Clean up existing policies""" for i in range(max_policies): - policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 3a59a41f..1bbbe4c2 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -92,8 +92,7 @@ Resources: - iam:PutRolePolicy Resource: - !Sub arn:aws:iam::${AWS::AccountId}:role/${IAMRoleName} - - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-iam-permissions-* - - !Sub arn:aws:iam::${AWS::AccountId}:policy/${IAMRoleName}-ManagedPolicy-* + - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-resource-collection-permissions-* - !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" DatadogAttachIntegrationPermissionsFunction: Type: AWS::Lambda::Function From ffdd521874fbe79dcaaa793e600415800971c9e9 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:28:35 -0400 Subject: [PATCH 14/32] fix typo --- aws_quickstart/attach_integration_permissions.py | 2 +- aws_quickstart/datadog_integration_role.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index b0b3bb2f..32b49c32 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -122,7 +122,7 @@ def attach_resource_collection_permissions(iam_client, role_name): # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy - policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_document = { "Version": "2012-10-17", "Statement": [ diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 1bbbe4c2..5caa3d4a 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -179,7 +179,7 @@ Resources: def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): """Clean up existing policies""" for i in range(max_policies): - policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION-{i+1}" policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( From af8bb4aa08a4c7151140c34638b36740098c5c4c Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:36:32 -0400 Subject: [PATCH 15/32] cleanup comments --- .../attach_integration_permissions.py | 14 +++++------ aws_quickstart/datadog_integration_role.yaml | 24 +++++++------------ 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 32b49c32..7a8c00e7 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -23,7 +23,7 @@ def create_chunks(permissions): current_chunk = [] for permission in permissions: - # Calculate size if we add this permission + # Determine policy size if we add this permission next_chunk = current_chunk + [permission] policy = { "Version": "2012-10-17", @@ -77,7 +77,6 @@ def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10 PolicyArn=policy_arn ) except iam_client.exceptions.NoSuchEntityException: - # Policy to detach doesn't exist pass except Exception as e: LOGGER.error(f"Error detaching policy {policy_name}: {str(e)}") @@ -86,11 +85,14 @@ def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10 iam_client.delete_policy( PolicyArn=policy_arn ) - except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): - # Policy to delete doesn't exist + iam_client.delete_role_policy( + RoleName=role_name, + PolicyName=POLICY_NAME_STANDARD + ) + except iam_client.exceptions.NoSuchEntityException: pass except Exception as e: - LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + LOGGER.error(f"Error deleting policy: {str(e)}") def attach_standard_permissions(iam_client, role_name): permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) @@ -111,9 +113,7 @@ def attach_standard_permissions(iam_client, role_name): PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) - def attach_resource_collection_permissions(iam_client, role_name): - # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) permission_chunks = create_chunks(permissions) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 5caa3d4a..965f66f2 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -133,7 +133,7 @@ Resources: current_chunk = [] for permission in permissions: - # Calculate size if we add this permission + # Determine policy size if we add this permission next_chunk = current_chunk + [permission] policy = { "Version": "2012-10-17", @@ -179,7 +179,7 @@ Resources: def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): """Clean up existing policies""" for i in range(max_policies): - policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION-{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( @@ -187,7 +187,6 @@ Resources: PolicyArn=policy_arn ) except iam_client.exceptions.NoSuchEntityException: - # Policy to detach doesn't exist pass except Exception as e: LOGGER.error(f"Error detaching policy {policy_name}: {str(e)}") @@ -196,11 +195,14 @@ Resources: iam_client.delete_policy( PolicyArn=policy_arn ) - except (iam_client.exceptions.NoSuchEntityException, iam_client.exceptions.DeleteConflictException): - # Policy to delete doesn't exist + iam_client.delete_role_policy( + RoleName=role_name, + PolicyName=POLICY_NAME_STANDARD + ) + except iam_client.exceptions.NoSuchEntityException: pass except Exception as e: - LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + LOGGER.error(f"Error deleting policy: {str(e)}") def attach_standard_permissions(iam_client, role_name): permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) @@ -221,9 +223,7 @@ Resources: PolicyDocument=json.dumps(policy_document, separators=(',', ':')) ) - def attach_resource_collection_permissions(iam_client, role_name): - # Fetch and smart chunk permissions based on character limits permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) permission_chunks = create_chunks(permissions) @@ -232,7 +232,7 @@ Resources: # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy - policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-part{i+1}" + policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_document = { "Version": "2012-10-17", "Statement": [ @@ -243,12 +243,9 @@ Resources: } ] } - - # Verify policy size before creating policy_json = json.dumps(policy_document, separators=(',', ':')) policy_size = len(policy_json) LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - policy = iam_client.create_policy( PolicyName=policy_name, PolicyDocument=policy_json @@ -292,7 +289,6 @@ Resources: handle_delete(event, context, role_name, account_id) else: handle_create_update(event, context, role_name, account_id) - DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger Condition: ShouldInstallSecurityAuditPolicy @@ -300,8 +296,6 @@ Resources: ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn DatadogIntegrationRole: !Ref IAMRoleName AccountId: !Ref AWS::AccountId - Partition: !Ref AWS::Partition - ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: ParameterGroups: From 83265b3aaa6a726e61574964471e200793522c6e Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:52:32 -0400 Subject: [PATCH 16/32] only attach resource collection permissions when specified --- aws_quickstart/attach_integration_permissions.py | 11 +++++------ aws_quickstart/datadog_integration_role.yaml | 13 ++++++------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 7a8c00e7..007182cd 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -133,12 +133,9 @@ def attach_resource_collection_permissions(iam_client, role_name): } ] } - - # Verify policy size before creating policy_json = json.dumps(policy_document, separators=(',', ':')) policy_size = len(policy_json) LOGGER.info(f"Creating policy {policy_name} with {len(chunk)} permissions ({policy_size} characters)") - policy = iam_client.create_policy( PolicyName=policy_name, PolicyDocument=policy_json @@ -160,13 +157,14 @@ def handle_delete(event, context, role_name, account_id): LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) -def handle_create_update(event, context, role_name, account_id): +def handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id) attach_standard_permissions(iam_client, role_name) - attach_resource_collection_permissions(iam_client, role_name) + if should_install_security_audit_policy: + attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") @@ -177,8 +175,9 @@ def handler(event, context): role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] + should_install_security_audit_policy = event['ResourceProperties']['ShouldInstallSecurityAuditPolicy'] if event['RequestType'] == 'Delete': handle_delete(event, context, role_name, account_id) else: - handle_create_update(event, context, role_name, account_id) + handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 965f66f2..93d3519e 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -61,10 +61,8 @@ Resources: [!Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit"], !Ref AWS::NoValue, ] - # Lambda function to dynamically attach permissions from DataDog API DatadogAttachIntegrationPermissionsLambdaExecutionRole: Type: AWS::IAM::Role - # Condition: ShouldInstallSecurityAuditPolicy Properties: AssumeRolePolicyDocument: Version: '2012-10-17' @@ -96,7 +94,6 @@ Resources: - !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" DatadogAttachIntegrationPermissionsFunction: Type: AWS::Lambda::Function - Condition: ShouldInstallSecurityAuditPolicy Properties: Description: "A function to attach Datadog AWS integration permissions to an IAM role." Role: !GetAtt DatadogAttachIntegrationPermissionsLambdaExecutionRole.Arn @@ -267,13 +264,14 @@ Resources: LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) - def handle_create_update(event, context, role_name, account_id): + def handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') cleanup_existing_policies(iam_client, role_name, account_id) attach_standard_permissions(iam_client, role_name) - attach_resource_collection_permissions(iam_client, role_name) + if should_install_security_audit_policy: + attach_resource_collection_permissions(iam_client, role_name) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error creating/attaching policy: {str(e)}") @@ -284,18 +282,19 @@ Resources: role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] + should_install_security_audit_policy = event['ResourceProperties']['ShouldInstallSecurityAuditPolicy'] if event['RequestType'] == 'Delete': handle_delete(event, context, role_name, account_id) else: - handle_create_update(event, context, role_name, account_id) + handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy) DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger - Condition: ShouldInstallSecurityAuditPolicy Properties: ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn DatadogIntegrationRole: !Ref IAMRoleName AccountId: !Ref AWS::AccountId + ShouldInstallSecurityAuditPolicy: !Ref ShouldInstallSecurityAuditPolicy Metadata: AWS::CloudFormation::Interface: ParameterGroups: From 06a2413c34b298909e8cbdd300d220cce8f4ed65 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 10:54:37 -0400 Subject: [PATCH 17/32] add lambda permission to delete inline policy: --- aws_quickstart/datadog_integration_role.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 93d3519e..fa03623f 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -85,6 +85,7 @@ Resources: Action: - iam:CreatePolicy - iam:DeletePolicy + - iam:DeleteRolePolicy - iam:AttachRolePolicy - iam:DetachRolePolicy - iam:PutRolePolicy From 9f0ff5a93f1340df0243aca7c5640e3a5403afe9 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 11:05:01 -0400 Subject: [PATCH 18/32] fix invalid parameter --- aws_quickstart/attach_integration_permissions.py | 2 +- aws_quickstart/datadog_integration_role.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 007182cd..8f1c8ad3 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -175,7 +175,7 @@ def handler(event, context): role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] - should_install_security_audit_policy = event['ResourceProperties']['ShouldInstallSecurityAuditPolicy'] + should_install_security_audit_policy = str(event['ResourceProperties']['ResourceCollectionPermissions']).lower() == 'true' if event['RequestType'] == 'Delete': handle_delete(event, context, role_name, account_id) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index fa03623f..075dc080 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -283,7 +283,7 @@ Resources: role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] - should_install_security_audit_policy = event['ResourceProperties']['ShouldInstallSecurityAuditPolicy'] + should_install_security_audit_policy = str(event['ResourceProperties']['ResourceCollectionPermissions']).lower() == 'true' if event['RequestType'] == 'Delete': handle_delete(event, context, role_name, account_id) @@ -295,7 +295,7 @@ Resources: ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn DatadogIntegrationRole: !Ref IAMRoleName AccountId: !Ref AWS::AccountId - ShouldInstallSecurityAuditPolicy: !Ref ShouldInstallSecurityAuditPolicy + ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: ParameterGroups: From b1d404339e06c426d6195e34c7607f54f5d004b9 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 11:09:51 -0400 Subject: [PATCH 19/32] bump version --- aws_quickstart/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index ff477c2a..1f69a7da 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v4.1.4 +v4.2.4 From 951625d67aa0147525bfac9dbf78fda818f40fa6 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 11:11:02 -0400 Subject: [PATCH 20/32] bump changelog --- aws_quickstart/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aws_quickstart/CHANGELOG.md b/aws_quickstart/CHANGELOG.md index a5f7630b..ab7eaa90 100644 --- a/aws_quickstart/CHANGELOG.md +++ b/aws_quickstart/CHANGELOG.md @@ -1,3 +1,8 @@ +# 4.2.4 (October 24, 2025) + +- Remove enumerated IAM permissions +- Add Lambda function to attach IAM permissions + # 4.1.4 (October 22, 2025) Add permissions to support resource collection for the following services: From f602af1324b9ad4362940fa0fcd364b29ca9d7dd Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 24 Oct 2025 11:18:26 -0400 Subject: [PATCH 21/32] update python runtime --- aws_quickstart/datadog_integration_role.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 075dc080..d7a05b15 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -102,7 +102,7 @@ Resources: LoggingConfig: ApplicationLogLevel: "INFO" LogFormat: "JSON" - Runtime: "python3.11" + Runtime: "python3.13" Timeout: 300 Code: ZipFile: | From 2532775a8ecd7fae152a1260474dbb1fc506aa07 Mon Sep 17 00:00:00 2001 From: Raymond Eah <78064236+raymondeah@users.noreply.github.com> Date: Fri, 24 Oct 2025 11:19:18 -0400 Subject: [PATCH 22/32] Update aws_quickstart/CHANGELOG.md Co-authored-by: Katie McKew <5915468+ktmq@users.noreply.github.com> --- aws_quickstart/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_quickstart/CHANGELOG.md b/aws_quickstart/CHANGELOG.md index ab7eaa90..a0bc4656 100644 --- a/aws_quickstart/CHANGELOG.md +++ b/aws_quickstart/CHANGELOG.md @@ -1,4 +1,4 @@ -# 4.2.4 (October 24, 2025) +# 4.2.0 (October 24, 2025) - Remove enumerated IAM permissions - Add Lambda function to attach IAM permissions From bb366e776f8ff2b37205017e7203e3be1340ec5b Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 09:17:02 -0500 Subject: [PATCH 23/32] use built in chunking in api layer --- aws_quickstart/datadog_integration_role.yaml | 43 ++------------------ 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index d7a05b15..e8278218 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -116,48 +116,14 @@ Resources: LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" - MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" - RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" + RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection?chunked=true" class DatadogAPIError(Exception): pass - def create_chunks(permissions): - """Create chunks of permissions based on character limit""" - chunks = [] - current_chunk = [] - - for permission in permissions: - # Determine policy size if we add this permission - next_chunk = current_chunk + [permission] - policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": next_chunk, - "Resource": "*" - } - ] - } - policy_size = len(json.dumps(policy, separators=(',', ':'))) - - # If adding this permission would exceed the limit, start a new chunk - if policy_size > MAX_POLICY_SIZE and current_chunk: - chunks.append(current_chunk) - current_chunk = [permission] - else: - current_chunk.append(permission) - - # Add the last chunk if it has permissions - if current_chunk: - chunks.append(current_chunk) - - return chunks - def fetch_permissions_from_datadog(api_url): """Fetch permissions from Datadog API""" headers = { @@ -222,11 +188,8 @@ Resources: ) def attach_resource_collection_permissions(iam_client, role_name): - permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - permission_chunks = create_chunks(permissions) - - LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - + permission_chunks = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy From 1f634504c388f0912fc527f1620863c9f5db0f7c Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 09:26:59 -0500 Subject: [PATCH 24/32] align python code --- .../attach_integration_permissions.py | 43 ++----------------- 1 file changed, 3 insertions(+), 40 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 8f1c8ad3..2b9ed3cf 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -8,48 +8,14 @@ LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" -MAX_POLICY_SIZE = 6144 # Maximum characters for AWS managed policy document POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" -RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection" +RESOURCE_COLLECTION_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/resource_collection?chunked=true" class DatadogAPIError(Exception): pass -def create_chunks(permissions): - """Create chunks of permissions based on character limit""" - chunks = [] - current_chunk = [] - - for permission in permissions: - # Determine policy size if we add this permission - next_chunk = current_chunk + [permission] - policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": next_chunk, - "Resource": "*" - } - ] - } - policy_size = len(json.dumps(policy, separators=(',', ':'))) - - # If adding this permission would exceed the limit, start a new chunk - if policy_size > MAX_POLICY_SIZE and current_chunk: - chunks.append(current_chunk) - current_chunk = [permission] - else: - current_chunk.append(permission) - - # Add the last chunk if it has permissions - if current_chunk: - chunks.append(current_chunk) - - return chunks - def fetch_permissions_from_datadog(api_url): """Fetch permissions from Datadog API""" headers = { @@ -114,11 +80,8 @@ def attach_standard_permissions(iam_client, role_name): ) def attach_resource_collection_permissions(iam_client, role_name): - permissions = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - permission_chunks = create_chunks(permissions) - - LOGGER.info(f"Created {len(permission_chunks)} policy chunks from {len(permissions)} permissions") - + permission_chunks = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) + # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy From 84e257e30da2556698ee5d3b574cb5be26785964 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 09:28:27 -0500 Subject: [PATCH 25/32] fix typo --- .../attach_integration_permissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_attach_integration_permissions/attach_integration_permissions.py b/aws_attach_integration_permissions/attach_integration_permissions.py index c8d1a260..3add5315 100644 --- a/aws_attach_integration_permissions/attach_integration_permissions.py +++ b/aws_attach_integration_permissions/attach_integration_permissions.py @@ -8,7 +8,7 @@ LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) -API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" +API_CALL_SOURCE_HEADER_VALUE = "cfn-quickstart" CHUNK_SIZE = 150 # Maximum number of IAM permissions per customer managed policy BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" From 611a30df0a22563368089ba26759a36f1f29aac3 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 09:33:50 -0500 Subject: [PATCH 26/32] fix typo --- .../attach_integration_permissions.py | 2 +- aws_quickstart/attach_integration_permissions.py | 2 +- aws_quickstart/datadog_integration_role.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws_attach_integration_permissions/attach_integration_permissions.py b/aws_attach_integration_permissions/attach_integration_permissions.py index 3add5315..c8d1a260 100644 --- a/aws_attach_integration_permissions/attach_integration_permissions.py +++ b/aws_attach_integration_permissions/attach_integration_permissions.py @@ -8,7 +8,7 @@ LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) -API_CALL_SOURCE_HEADER_VALUE = "cfn-quickstart" +API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" CHUNK_SIZE = 150 # Maximum number of IAM permissions per customer managed policy BASE_POLICY_PREFIX = "datadog-aws-integration-iam-permissions" diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 2b9ed3cf..fd3f4f93 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -7,7 +7,7 @@ LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) -API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" +API_CALL_SOURCE_HEADER_VALUE = "cfn-quickstart" POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index e8278218..335d3749 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -115,7 +115,7 @@ Resources: LOGGER = logging.getLogger() LOGGER.setLevel(logging.INFO) - API_CALL_SOURCE_HEADER_VALUE = "cfn-iam-permissions" + API_CALL_SOURCE_HEADER_VALUE = "cfn-quickstart" POLICY_NAME_STANDARD = "DatadogAWSIntegrationPolicy" BASE_POLICY_PREFIX_RESOURCE_COLLECTION = "datadog-aws-integration-resource-collection-permissions" STANDARD_PERMISSIONS_API_URL = "https://api.datadoghq.com/api/v2/integration/aws/iam_permissions/standard" @@ -189,7 +189,7 @@ Resources: def attach_resource_collection_permissions(iam_client, role_name): permission_chunks = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - + # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy From 3f92cb0f3b9652eff857059b8eb13ce3f79d7958 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 09:47:47 -0500 Subject: [PATCH 27/32] separate deletion behavior for standard and resource collection permissions --- .../attach_integration_permissions.py | 23 +++++++++++++------ aws_quickstart/datadog_integration_role.yaml | 21 ++++++++++++----- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index fd3f4f93..6d5507e1 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -33,7 +33,7 @@ def fetch_permissions_from_datadog(api_url): return json_response["data"]["attributes"]["permissions"] def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): - """Clean up existing policies""" + # Remove resource collection permissions for i in range(max_policies): policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" @@ -51,14 +51,23 @@ def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10 iam_client.delete_policy( PolicyArn=policy_arn ) - iam_client.delete_role_policy( - RoleName=role_name, - PolicyName=POLICY_NAME_STANDARD - ) except iam_client.exceptions.NoSuchEntityException: pass + except iam_client.exceptions.DeleteConflictException: + LOGGER.warning(f"Policy {policy_name} still attached, skipping delete") except Exception as e: - LOGGER.error(f"Error deleting policy: {str(e)}") + LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + + # Remove standard permissions + try: + iam_client.delete_role_policy( + RoleName=role_name, + PolicyName=POLICY_NAME_STANDARD + ) + except iam_client.exceptions.NoSuchEntityException: + pass + except Exception as e: + LOGGER.error(f"Error deleting inline policy {POLICY_NAME_STANDARD}: {str(e)}") def attach_standard_permissions(iam_client, role_name): permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) @@ -81,7 +90,7 @@ def attach_standard_permissions(iam_client, role_name): def attach_resource_collection_permissions(iam_client, role_name): permission_chunks = fetch_permissions_from_datadog(RESOURCE_COLLECTION_PERMISSIONS_API_URL) - + # Create and attach new policies for i, chunk in enumerate(permission_chunks): # Create policy diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 335d3749..f7ace1f7 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -141,7 +141,7 @@ Resources: return json_response["data"]["attributes"]["permissions"] def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): - """Clean up existing policies""" + # Remove resource collection permissions for i in range(max_policies): policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" @@ -159,14 +159,23 @@ Resources: iam_client.delete_policy( PolicyArn=policy_arn ) - iam_client.delete_role_policy( - RoleName=role_name, - PolicyName=POLICY_NAME_STANDARD - ) except iam_client.exceptions.NoSuchEntityException: pass + except iam_client.exceptions.DeleteConflictException: + LOGGER.warning(f"Policy {policy_name} still attached, skipping delete") except Exception as e: - LOGGER.error(f"Error deleting policy: {str(e)}") + LOGGER.error(f"Error deleting policy {policy_name}: {str(e)}") + + # Remove standard permissions + try: + iam_client.delete_role_policy( + RoleName=role_name, + PolicyName=POLICY_NAME_STANDARD + ) + except iam_client.exceptions.NoSuchEntityException: + pass + except Exception as e: + LOGGER.error(f"Error deleting inline policy {POLICY_NAME_STANDARD}: {str(e)}") def attach_standard_permissions(iam_client, role_name): permissions = fetch_permissions_from_datadog(STANDARD_PERMISSIONS_API_URL) From 468d5ea14421daea536f96e6a33b86c5d5e25b0e Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Fri, 9 Jan 2026 10:22:30 -0500 Subject: [PATCH 28/32] update changelog --- aws_quickstart/CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_quickstart/CHANGELOG.md b/aws_quickstart/CHANGELOG.md index 7f1e5559..3e3271e2 100644 --- a/aws_quickstart/CHANGELOG.md +++ b/aws_quickstart/CHANGELOG.md @@ -1,7 +1,7 @@ -# 4.4.0 (October 24, 2025) +# 4.4.0 (January 9, 2026) -- Remove enumerated IAM permissions -- Add Lambda function to attach IAM permissions +- Remove all enumerated IAM permissions from the datadog_integration_role stack +- Add Lambda function to retrieve and attach the equivalent IAM permissions at stack creation/update time # 4.3.0 (December 26, 2025) From b3e952280acf5d451672830fb736663147d725bf Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Mon, 9 Feb 2026 11:42:55 -0500 Subject: [PATCH 29/32] use aws partition when constructing ARNs --- aws_quickstart/datadog_integration_role.yaml | 25 +++++++++++--------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index f7ace1f7..7615be7b 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -90,8 +90,8 @@ Resources: - iam:DetachRolePolicy - iam:PutRolePolicy Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/${IAMRoleName} - - !Sub arn:aws:iam::${AWS::AccountId}:policy/datadog-aws-integration-resource-collection-permissions-* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/${IAMRoleName} + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/datadog-aws-integration-resource-collection-permissions-* - !Sub "arn:${AWS::Partition}:iam::aws:policy/SecurityAudit" DatadogAttachIntegrationPermissionsFunction: Type: AWS::Lambda::Function @@ -140,11 +140,11 @@ Resources: return json_response["data"]["attributes"]["permissions"] - def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): + def cleanup_existing_policies(iam_client, role_name, account_id, partition, max_policies=10): # Remove resource collection permissions for i in range(max_policies): policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" - policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" + policy_arn = f"arn:{partition}:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( RoleName=role_name, @@ -227,21 +227,21 @@ Resources: PolicyArn=policy['Policy']['Arn'] ) - def handle_delete(event, context, role_name, account_id): + def handle_delete(event, context, role_name, account_id, partition): """Handle stack deletion.""" iam_client = boto3.client('iam') try: - cleanup_existing_policies(iam_client, role_name, account_id) + cleanup_existing_policies(iam_client, role_name, account_id, partition) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) - def handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy): + def handle_create_update(event, context, role_name, account_id, partition, should_install_security_audit_policy): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') - cleanup_existing_policies(iam_client, role_name, account_id) + cleanup_existing_policies(iam_client, role_name, account_id, partition) attach_standard_permissions(iam_client, role_name) if should_install_security_audit_policy: attach_resource_collection_permissions(iam_client, role_name) @@ -255,18 +255,21 @@ Resources: role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] + partition = event['ResourceProperties'].get('Partition', 'aws') should_install_security_audit_policy = str(event['ResourceProperties']['ResourceCollectionPermissions']).lower() == 'true' - + if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id) + handle_delete(event, context, role_name, account_id, partition) else: - handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy) + handle_create_update(event, context, role_name, account_id, partition, should_install_security_audit_policy) DatadogAttachIntegrationPermissionsFunctionTrigger: Type: Custom::DatadogAttachIntegrationPermissionsFunctionTrigger + DependsOn: DatadogIntegrationRole Properties: ServiceToken: !GetAtt DatadogAttachIntegrationPermissionsFunction.Arn DatadogIntegrationRole: !Ref IAMRoleName AccountId: !Ref AWS::AccountId + Partition: !Sub "${AWS::Partition}" ResourceCollectionPermissions: !Ref ResourceCollectionPermissions Metadata: AWS::CloudFormation::Interface: From 5c632ca810aec6de40994864676b77541c0ad816 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Mon, 9 Feb 2026 11:45:15 -0500 Subject: [PATCH 30/32] fix version numbers --- aws_quickstart/CHANGELOG.md | 2 +- aws_quickstart/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_quickstart/CHANGELOG.md b/aws_quickstart/CHANGELOG.md index 3e3271e2..ab5ae8ce 100644 --- a/aws_quickstart/CHANGELOG.md +++ b/aws_quickstart/CHANGELOG.md @@ -1,4 +1,4 @@ -# 4.4.0 (January 9, 2026) +# 4.4.0 (February 9, 2026) - Remove all enumerated IAM permissions from the datadog_integration_role stack - Add Lambda function to retrieve and attach the equivalent IAM permissions at stack creation/update time diff --git a/aws_quickstart/version.txt b/aws_quickstart/version.txt index c2d2cb0b..37c93bef 100644 --- a/aws_quickstart/version.txt +++ b/aws_quickstart/version.txt @@ -1 +1 @@ -v4.5.0 +v4.4.0 From 41fda055f929ea0ec251bac86107332ce88a53e3 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Mon, 9 Feb 2026 12:03:10 -0500 Subject: [PATCH 31/32] sync both python codes --- .../attach_integration_permissions.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 6d5507e1..3c057dc5 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -32,11 +32,11 @@ def fetch_permissions_from_datadog(api_url): return json_response["data"]["attributes"]["permissions"] -def cleanup_existing_policies(iam_client, role_name, account_id, max_policies=10): +def cleanup_existing_policies(iam_client, role_name, account_id, partition, max_policies=10): # Remove resource collection permissions for i in range(max_policies): policy_name = f"{BASE_POLICY_PREFIX_RESOURCE_COLLECTION}-{i+1}" - policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" + policy_arn = f"arn:{partition}:iam::{account_id}:policy/{policy_name}" try: iam_client.detach_role_policy( RoleName=role_name, @@ -119,21 +119,21 @@ def attach_resource_collection_permissions(iam_client, role_name): PolicyArn=policy['Policy']['Arn'] ) -def handle_delete(event, context, role_name, account_id): +def handle_delete(event, context, role_name, account_id, partition): """Handle stack deletion.""" iam_client = boto3.client('iam') try: - cleanup_existing_policies(iam_client, role_name, account_id) + cleanup_existing_policies(iam_client, role_name, account_id, partition) cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData={}) except Exception as e: LOGGER.error(f"Error deleting policy: {str(e)}") cfnresponse.send(event, context, cfnresponse.FAILED, responseData={"Message": str(e)}) -def handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy): +def handle_create_update(event, context, role_name, account_id, partition, should_install_security_audit_policy): """Handle stack creation or update.""" try: iam_client = boto3.client('iam') - cleanup_existing_policies(iam_client, role_name, account_id) + cleanup_existing_policies(iam_client, role_name, account_id, partition) attach_standard_permissions(iam_client, role_name) if should_install_security_audit_policy: attach_resource_collection_permissions(iam_client, role_name) @@ -147,9 +147,10 @@ def handler(event, context): role_name = event['ResourceProperties']['DatadogIntegrationRole'] account_id = event['ResourceProperties']['AccountId'] + partition = event['ResourceProperties'].get('Partition', 'aws') should_install_security_audit_policy = str(event['ResourceProperties']['ResourceCollectionPermissions']).lower() == 'true' - + if event['RequestType'] == 'Delete': - handle_delete(event, context, role_name, account_id) + handle_delete(event, context, role_name, account_id, partition) else: - handle_create_update(event, context, role_name, account_id, should_install_security_audit_policy) + handle_create_update(event, context, role_name, account_id, partition, should_install_security_audit_policy) From a1fb1b780a9f64ee51deea67ec22a28b46297ce8 Mon Sep 17 00:00:00 2001 From: Raymond Eah Date: Mon, 9 Feb 2026 12:05:00 -0500 Subject: [PATCH 32/32] add proper try catch to api request error handling --- aws_quickstart/attach_integration_permissions.py | 15 +++++++++------ aws_quickstart/datadog_integration_role.yaml | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/aws_quickstart/attach_integration_permissions.py b/aws_quickstart/attach_integration_permissions.py index 3c057dc5..16bcc76e 100644 --- a/aws_quickstart/attach_integration_permissions.py +++ b/aws_quickstart/attach_integration_permissions.py @@ -1,7 +1,8 @@ import json import logging from urllib.request import Request -import urllib +import urllib.error +import urllib.request import cfnresponse import boto3 @@ -24,12 +25,14 @@ def fetch_permissions_from_datadog(api_url): request = Request(api_url, headers=headers) request.get_method = lambda: "GET" - response = urllib.request.urlopen(request) + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + error_body = json.loads(e.read()) + error_message = error_body.get('errors', ['Unknown error'])[0] + raise DatadogAPIError(f"Datadog API error: {error_message}") from e + json_response = json.loads(response.read()) - if response.getcode() != 200: - error_message = json_response.get('errors', ['Unknown error'])[0] - raise DatadogAPIError(f"Datadog API error: {error_message}") - return json_response["data"]["attributes"]["permissions"] def cleanup_existing_policies(iam_client, role_name, account_id, partition, max_policies=10): diff --git a/aws_quickstart/datadog_integration_role.yaml b/aws_quickstart/datadog_integration_role.yaml index 7615be7b..17fb7928 100644 --- a/aws_quickstart/datadog_integration_role.yaml +++ b/aws_quickstart/datadog_integration_role.yaml @@ -109,7 +109,8 @@ Resources: import json import logging from urllib.request import Request - import urllib + import urllib.error + import urllib.request import cfnresponse import boto3 @@ -132,12 +133,14 @@ Resources: request = Request(api_url, headers=headers) request.get_method = lambda: "GET" - response = urllib.request.urlopen(request) + try: + response = urllib.request.urlopen(request) + except urllib.error.HTTPError as e: + error_body = json.loads(e.read()) + error_message = error_body.get('errors', ['Unknown error'])[0] + raise DatadogAPIError(f"Datadog API error: {error_message}") from e + json_response = json.loads(response.read()) - if response.getcode() != 200: - error_message = json_response.get('errors', ['Unknown error'])[0] - raise DatadogAPIError(f"Datadog API error: {error_message}") - return json_response["data"]["attributes"]["permissions"] def cleanup_existing_policies(iam_client, role_name, account_id, partition, max_policies=10):