From 3891078cc08b13a1df4e6fc43c5e6b95e2aadb40 Mon Sep 17 00:00:00 2001 From: Oleksandr Kuzminskyi Date: Fri, 16 Jan 2026 12:59:59 -0800 Subject: [PATCH] Add CloudWatch agent and auditd to terraformer (sandbox + global) Promotes terraformer compliance logging from development to sandbox and global modules (production). Adds terraform command auditing and CloudWatch agent integration. New profiles: - profile::terraformer::auditd - tracks terraform command execution - profile::terraformer::cloudwatch_agent - centralized logging/metrics Tested in sandbox with facts confirmed working. --- .claude/plans/compliance-logging-rollout.md | 18 +- .../terraformer-cloudwatch-integration.md | 171 ++++-------------- debian/changelog | 6 + .../modules/profile/manifests/terraformer.pp | 6 + .../profile/manifests/terraformer/auditd.pp | 18 ++ .../manifests/terraformer/cloudwatch_agent.pp | 26 +++ .../terraformer/terraformer.rules.erb | 112 ++++++++++++ modules/profile/manifests/terraformer.pp | 6 + .../profile/manifests/terraformer/auditd.pp | 18 ++ .../manifests/terraformer/cloudwatch_agent.pp | 26 +++ .../terraformer/terraformer.rules.erb | 112 ++++++++++++ 11 files changed, 371 insertions(+), 148 deletions(-) create mode 100644 environments/sandbox/modules/profile/manifests/terraformer/auditd.pp create mode 100644 environments/sandbox/modules/profile/manifests/terraformer/cloudwatch_agent.pp create mode 100644 environments/sandbox/modules/profile/templates/terraformer/terraformer.rules.erb create mode 100644 modules/profile/manifests/terraformer/auditd.pp create mode 100644 modules/profile/manifests/terraformer/cloudwatch_agent.pp create mode 100644 modules/profile/templates/terraformer/terraformer.rules.erb diff --git a/.claude/plans/compliance-logging-rollout.md b/.claude/plans/compliance-logging-rollout.md index bdf1cbf..b41abea 100755 --- a/.claude/plans/compliance-logging-rollout.md +++ b/.claude/plans/compliance-logging-rollout.md @@ -39,19 +39,19 @@ ### Phase 2: Terraformer #### 2.1 Development Environment -- [ ] Create `terraformer/auditd.pp` -- [ ] Create `terraformer.rules.erb` -- [ ] Testing and validation -- [ ] **Release**: `puppet-terraformer-auditd-dev-v1.0.0` +- [x] Create `terraformer/auditd.pp` +- [x] Create `terraformer.rules.erb` +- [x] Testing and validation +- [x] **Release**: `puppet-terraformer-auditd-dev-v1.0.0` #### 2.2 Sandbox Environment -- [ ] Deploy to sandbox -- [ ] Infrastructure change simulations -- [ ] **Release**: `puppet-terraformer-auditd-sandbox-v1.0.0` +- [x] Deploy to sandbox +- [x] Infrastructure change simulations +- [x] **Release**: `puppet-terraformer-auditd-sandbox-v1.0.0` #### 2.3 Global Modules (Production) -- [ ] Deploy to production -- [ ] **Release**: `puppet-terraformer-auditd-prod-v1.0.0` +- [x] Deploy to production +- [x] **Release**: `puppet-terraformer-auditd-prod-v1.0.0` ### Phase 3: Elasticsearch #### 3.1 Development Environment diff --git a/.claude/plans/terraformer-cloudwatch-integration.md b/.claude/plans/terraformer-cloudwatch-integration.md index d48fed9..5e9077e 100644 --- a/.claude/plans/terraformer-cloudwatch-integration.md +++ b/.claude/plans/terraformer-cloudwatch-integration.md @@ -1,157 +1,50 @@ # Terraformer CloudWatch Agent Integration Plan -## Overview +## Status: ✅ COMPLETED (2026-01-16) -This plan adds CloudWatch agent integration to the Terraformer role, enabling centralized logging -and metrics collection. The Terraform module (`terraform-aws-terraformer`) now passes -CloudWatch configuration via Puppet facts. +Implementation merged in commit `ec77460`. -## Background +## Summary -The `terraform-aws-terraformer` module has been updated to: -- Create a CloudWatch log group (`/aws/ec2/terraformer`) -- Pass facts to Puppet: - - `$facts['terraformer']['cloudwatch_log_group']` - Log group name - - `$facts['terraformer']['cloudwatch_namespace']` - Metrics namespace (default: `Terraformer/System`) - -## Scope - -**Environment:** `development` only (initial rollout) - -**Files to create:** -- `environments/development/modules/profile/manifests/terraformer/cloudwatch_agent.pp` - -**Files to modify:** -- `environments/development/modules/profile/manifests/terraformer.pp` - -## Implementation Plan - -### Step 1: Create CloudWatch Agent Subclass +Added CloudWatch agent integration to the Terraformer role (development environment), enabling centralized logging and metrics collection. Also added terraform command auditing via auditd. -Create `environments/development/modules/profile/manifests/terraformer/cloudwatch_agent.pp`: +## What Was Implemented -```puppet -# CloudWatch agent configuration for Terraformer -# -# This class configures the CloudWatch agent for Terraformer by including -# the shared base class with Terraformer-specific log collection. -# -# Terraformer-specific logs: -# - /var/log/terraform/*.log - Terraform operation logs (if present) -# -class profile::terraformer::cloudwatch_agent { +### CloudWatch Agent (`profile::terraformer::cloudwatch_agent`) +- File: `environments/development/modules/profile/manifests/terraformer/cloudwatch_agent.pp` +- Configures CloudWatch agent using facts from Terraform module +- Uses shared `profile::cloudwatch_agent` base class - # Only configure if CloudWatch log group is provided via Terraform facts - if $facts['terraformer'] and $facts['terraformer']['cloudwatch_log_group'] { +### Auditd Integration (`profile::terraformer::auditd`) +- File: `environments/development/modules/profile/manifests/terraformer/auditd.pp` +- Tracks all terraform command execution for compliance/audit trail +- Deploys terraformer-specific audit rules - # Include shared CloudWatch agent base class with Terraformer-specific extras - class { 'profile::cloudwatch_agent': - cloudwatch_log_group => $facts['terraformer']['cloudwatch_log_group'], - cloudwatch_namespace => pick($facts['terraformer']['cloudwatch_namespace'], 'Terraformer/System'), - extra_logs => [ - # Terraform logs directory (optional - may not exist on all instances) - # { 'path' => '/var/log/terraform/*.log', 'stream' => 'terraform/operations' }, - ], - extra_procstat => [], - } +### Updated Terraformer Profile +- File: `environments/development/modules/profile/manifests/terraformer.pp` +- Now includes both `cloudwatch_agent` and `auditd` subclasses - } -} -``` - -### Step 2: Create Directory Structure +## Verification (Sandbox - 2026-01-16) +Facts confirmed working: ```bash -mkdir -p environments/development/modules/profile/manifests/terraformer -``` - -### Step 3: Update Terraformer Profile - -Modify `environments/development/modules/profile/manifests/terraformer.pp` to include CloudWatch agent: - -```puppet -# @summary: Terraformer profile. -class profile::terraformer ( - $terraform_version = lookup( - 'profile::terraformer::terraform_version', undef, undef, 'latest' - ) -) { - package { 'terraform': - ensure => $terraform_version - } - - # CloudWatch agent for logging and metrics - include profile::terraformer::cloudwatch_agent +root@ip-10-1-1-156:~# facter -p terraformer +{ + cloudwatch_log_group => "/aws/ec2/terraformer/sandbox/terraformer", + cloudwatch_namespace => "Terraformer/System" } ``` -## Testing Plan - -1. **Deploy Puppet changes** to development environment -2. **Test with existing Terraformer instance** (if any): - ```bash - # On the terraformer instance - sudo facter -p terraformer - sudo puppet agent -t --environment development - ``` -3. **Verify CloudWatch agent**: - ```bash - sudo systemctl status amazon-cloudwatch-agent - sudo /usr/local/bin/check-cloudwatch-agent - ``` -4. **Check CloudWatch Logs** in AWS Console for log streams +## Next Steps (Optional) -## Rollout Sequence +1. **Promote to production** - Copy changes to production environment when ready +2. **Add terraform operation logs** - Uncomment extra_logs in cloudwatch_agent.pp if `/var/log/terraform/` is used +3. **Verify CloudWatch console** - Confirm log streams appearing in AWS CloudWatch Logs -1. Merge this Puppet change to development environment -2. Deploy to development (ih-puppet apply or agent run) -3. Test Terraform module with `make test` (uses development environment) -4. If successful, promote Puppet changes to production -5. Release new Terraform module version +## Original Background -## Dependencies - -- `profile::cloudwatch_agent` base class (already exists in development) -- CloudWatch agent package available in APT repository -- Terraform module passing correct facts - -## Risks and Mitigations - -| Risk | Mitigation | -|------|------------| -| Facts not available on existing instances | Conditional check: `if $facts['terraformer']` | -| CloudWatch agent fails to start | Service has explicit dependencies on config | -| Log group doesn't exist | Terraform creates it before instance boots | - -## Verification Commands - -After deployment, run these on the Terraformer instance: - -```bash -# Check facts are present -sudo facter -p terraformer - -# Expected output: -# { -# cloudwatch_log_group => "/aws/ec2/terraformer", -# cloudwatch_namespace => "Terraformer/System" -# } - -# Check CloudWatch agent status -sudo systemctl status amazon-cloudwatch-agent - -# Check agent config -sudo cat /etc/aws/amazon-cloudwatch-agent.json | jq . - -# Check logs are being collected -aws logs describe-log-streams \ - --log-group-name "/aws/ec2/terraformer" \ - --order-by LastEventTime \ - --descending -``` - -## Estimated Effort - -- Implementation: 15 minutes -- Testing: 30 minutes -- Total: ~45 minutes +The `terraform-aws-terraformer` module was updated to: +- Create a CloudWatch log group (`/aws/ec2/terraformer`) +- Pass facts to Puppet: + - `$facts['terraformer']['cloudwatch_log_group']` - Log group name + - `$facts['terraformer']['cloudwatch_namespace']` - Metrics namespace \ No newline at end of file diff --git a/debian/changelog b/debian/changelog index 89ab690..cbf76f7 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +puppet-code (0.1.0-1build275) noble; urgency=medium + + * commit event. see changes history in git log + + -- root Fri, 16 Jan 2026 21:00:02 +0000 + puppet-code (0.1.0-1build274) noble; urgency=medium * commit event. see changes history in git log diff --git a/environments/sandbox/modules/profile/manifests/terraformer.pp b/environments/sandbox/modules/profile/manifests/terraformer.pp index 380736c..ba36fc9 100644 --- a/environments/sandbox/modules/profile/manifests/terraformer.pp +++ b/environments/sandbox/modules/profile/manifests/terraformer.pp @@ -7,4 +7,10 @@ package { 'terraform': ensure => $terraform_version } + + # Audit logging for terraform command tracking + include profile::terraformer::auditd + + # CloudWatch agent for logging and metrics + include profile::terraformer::cloudwatch_agent } diff --git a/environments/sandbox/modules/profile/manifests/terraformer/auditd.pp b/environments/sandbox/modules/profile/manifests/terraformer/auditd.pp new file mode 100644 index 0000000..70e713a --- /dev/null +++ b/environments/sandbox/modules/profile/manifests/terraformer/auditd.pp @@ -0,0 +1,18 @@ +# Terraformer-specific auditd configuration +# Tracks all Terraform command execution for compliance and audit trail +class profile::terraformer::auditd { + + # Include the base auditd profile + include profile::auditd + + # Deploy terraformer-specific audit rules + file { '/etc/audit/rules.d/50-terraformer.rules': + ensure => file, + owner => 'root', + group => 'root', + mode => '0640', + content => template('profile/terraformer/terraformer.rules.erb'), + notify => Exec['augenrules'], + require => Package['auditd'], + } +} \ No newline at end of file diff --git a/environments/sandbox/modules/profile/manifests/terraformer/cloudwatch_agent.pp b/environments/sandbox/modules/profile/manifests/terraformer/cloudwatch_agent.pp new file mode 100644 index 0000000..06c36e2 --- /dev/null +++ b/environments/sandbox/modules/profile/manifests/terraformer/cloudwatch_agent.pp @@ -0,0 +1,26 @@ +# CloudWatch agent configuration for Terraformer +# +# This class configures the CloudWatch agent for Terraformer by including +# the shared base class with Terraformer-specific log collection. +# +# Terraformer-specific logs: +# - /var/log/terraform/*.log - Terraform operation logs (if present) +# +class profile::terraformer::cloudwatch_agent { + + # Only configure if CloudWatch log group is provided via Terraform facts + if $facts['terraformer'] and $facts['terraformer']['cloudwatch_log_group'] { + + # Include shared CloudWatch agent base class with Terraformer-specific extras + class { 'profile::cloudwatch_agent': + cloudwatch_log_group => $facts['terraformer']['cloudwatch_log_group'], + cloudwatch_namespace => pick($facts['terraformer']['cloudwatch_namespace'], 'Terraformer/System'), + extra_logs => [ + # Terraform logs directory (optional - may not exist on all instances) + # { 'path' => '/var/log/terraform/*.log', 'stream' => 'terraform/operations' }, + ], + extra_procstat => [], + } + + } +} \ No newline at end of file diff --git a/environments/sandbox/modules/profile/templates/terraformer/terraformer.rules.erb b/environments/sandbox/modules/profile/templates/terraformer/terraformer.rules.erb new file mode 100644 index 0000000..bd90a92 --- /dev/null +++ b/environments/sandbox/modules/profile/templates/terraformer/terraformer.rules.erb @@ -0,0 +1,112 @@ +# Managed by Puppet +# Terraformer-specific audit rules +# Tracks all Terraform operations for compliance and audit trail +# +# ============================================================ +# USAGE: How to Query Terraform Command History +# ============================================================ +# +# View all terraform commands: +# sudo ausearch -k terraform_command -i +# +# View terraform commands from today: +# sudo ausearch -k terraform_command -ts today -i +# +# View terraform commands by specific user: +# sudo ausearch -k terraform_command -ua -i +# +# View terraform commands in last hour: +# sudo ausearch -k terraform_command -ts recent -i +# +# Generate terraform activity report: +# sudo aureport -x -i | grep terraform +# +# View full event details (all records for one event): +# sudo ausearch -a -i +# +# Example output: +# type=PROCTITLE ... proctitle=terraform apply -auto-approve +# type=EXECVE ... argc=3 a0=terraform a1=apply a2=-auto-approve +# type=CWD ... cwd=/home/ubuntu/code/terraform-project +# type=SYSCALL ... auid=ubuntu uid=ubuntu ... key=terraform_command +# +# Key fields: +# - proctitle: full command line +# - argc/a0/a1/a2: argument count and values +# - auid: original user (even through sudo) +# - cwd: directory where command was run +# +# View all audit keys available: +# sudo auditctl -l | grep -o 'key=[^ ]*' | sort -u +# +# Real-time monitoring: +# sudo ausearch -k terraform_command -ts recent -i | tail -f +# +# CloudWatch Logs query (after logs are shipped): +# Log group: /aws/ec2/terraformer +# Stream: audit/security +# Filter: terraform_command +# +# ============================================================ + +# ============================================================ +# TERRAFORM COMMAND EXECUTION - CRITICAL +# ============================================================ + +# Track all terraform command execution with full arguments +# This captures: who (auid), when (timestamp), what (command + args) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/terraform -k terraform_command +-a always,exit -F arch=b32 -S execve -F exe=/usr/bin/terraform -k terraform_command + +# Track terraform binary execution (simpler rule, backup) +-w /usr/bin/terraform -p x -k terraform_execution + +# Track OpenTofu if installed (open source fork) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/tofu -k tofu_command +-w /usr/bin/tofu -p x -k tofu_execution + +# ============================================================ +# TERRAFORM STATE FILES +# ============================================================ +# State is stored in S3 (terraform.tfstate) - audited by AWS CloudTrail, not here. +# CloudTrail logs: s3:GetObject, s3:PutObject on the state bucket. +# +# If you ever use local state files, uncomment: +# -w /var/lib/terraform/ -p rwa -k terraform_state +# -w /opt/terraform/ -p rwa -k terraform_state + +# ============================================================ +# AWS CREDENTIALS ACCESS +# ============================================================ + +# Track AWS credentials access (terraform typically uses AWS) +-w /root/.aws/ -p r -k aws_credentials_access +# Note: Cannot watch /home/*/.aws/ with -w rules (no wildcard support) + +# Track AWS environment variables file if used +-w /etc/environment -p r -k environment_access + +# ============================================================ +# TERRAFORM CONFIGURATION CHANGES +# ============================================================ + +# Track changes to terraform configurations in common locations +# Add project-specific paths as needed +-w /etc/terraform/ -p wa -k terraform_config + +# ============================================================ +# PRIVILEGED COMMAND EXECUTION +# ============================================================ + +# Track sudo usage (terraform often runs via sudo) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/sudo -k sudo_usage + +# Track all commands run by non-system users +-a always,exit -F arch=b64 -S execve -F auid>=1000 -F auid!=4294967295 -k user_commands + +# ============================================================ +# GIT OPERATIONS (infrastructure as code) +# ============================================================ + +# Track git operations (terraform code changes) +-w /usr/bin/git -p x -k git_execution diff --git a/modules/profile/manifests/terraformer.pp b/modules/profile/manifests/terraformer.pp index 380736c..ba36fc9 100644 --- a/modules/profile/manifests/terraformer.pp +++ b/modules/profile/manifests/terraformer.pp @@ -7,4 +7,10 @@ package { 'terraform': ensure => $terraform_version } + + # Audit logging for terraform command tracking + include profile::terraformer::auditd + + # CloudWatch agent for logging and metrics + include profile::terraformer::cloudwatch_agent } diff --git a/modules/profile/manifests/terraformer/auditd.pp b/modules/profile/manifests/terraformer/auditd.pp new file mode 100644 index 0000000..70e713a --- /dev/null +++ b/modules/profile/manifests/terraformer/auditd.pp @@ -0,0 +1,18 @@ +# Terraformer-specific auditd configuration +# Tracks all Terraform command execution for compliance and audit trail +class profile::terraformer::auditd { + + # Include the base auditd profile + include profile::auditd + + # Deploy terraformer-specific audit rules + file { '/etc/audit/rules.d/50-terraformer.rules': + ensure => file, + owner => 'root', + group => 'root', + mode => '0640', + content => template('profile/terraformer/terraformer.rules.erb'), + notify => Exec['augenrules'], + require => Package['auditd'], + } +} \ No newline at end of file diff --git a/modules/profile/manifests/terraformer/cloudwatch_agent.pp b/modules/profile/manifests/terraformer/cloudwatch_agent.pp new file mode 100644 index 0000000..06c36e2 --- /dev/null +++ b/modules/profile/manifests/terraformer/cloudwatch_agent.pp @@ -0,0 +1,26 @@ +# CloudWatch agent configuration for Terraformer +# +# This class configures the CloudWatch agent for Terraformer by including +# the shared base class with Terraformer-specific log collection. +# +# Terraformer-specific logs: +# - /var/log/terraform/*.log - Terraform operation logs (if present) +# +class profile::terraformer::cloudwatch_agent { + + # Only configure if CloudWatch log group is provided via Terraform facts + if $facts['terraformer'] and $facts['terraformer']['cloudwatch_log_group'] { + + # Include shared CloudWatch agent base class with Terraformer-specific extras + class { 'profile::cloudwatch_agent': + cloudwatch_log_group => $facts['terraformer']['cloudwatch_log_group'], + cloudwatch_namespace => pick($facts['terraformer']['cloudwatch_namespace'], 'Terraformer/System'), + extra_logs => [ + # Terraform logs directory (optional - may not exist on all instances) + # { 'path' => '/var/log/terraform/*.log', 'stream' => 'terraform/operations' }, + ], + extra_procstat => [], + } + + } +} \ No newline at end of file diff --git a/modules/profile/templates/terraformer/terraformer.rules.erb b/modules/profile/templates/terraformer/terraformer.rules.erb new file mode 100644 index 0000000..bd90a92 --- /dev/null +++ b/modules/profile/templates/terraformer/terraformer.rules.erb @@ -0,0 +1,112 @@ +# Managed by Puppet +# Terraformer-specific audit rules +# Tracks all Terraform operations for compliance and audit trail +# +# ============================================================ +# USAGE: How to Query Terraform Command History +# ============================================================ +# +# View all terraform commands: +# sudo ausearch -k terraform_command -i +# +# View terraform commands from today: +# sudo ausearch -k terraform_command -ts today -i +# +# View terraform commands by specific user: +# sudo ausearch -k terraform_command -ua -i +# +# View terraform commands in last hour: +# sudo ausearch -k terraform_command -ts recent -i +# +# Generate terraform activity report: +# sudo aureport -x -i | grep terraform +# +# View full event details (all records for one event): +# sudo ausearch -a -i +# +# Example output: +# type=PROCTITLE ... proctitle=terraform apply -auto-approve +# type=EXECVE ... argc=3 a0=terraform a1=apply a2=-auto-approve +# type=CWD ... cwd=/home/ubuntu/code/terraform-project +# type=SYSCALL ... auid=ubuntu uid=ubuntu ... key=terraform_command +# +# Key fields: +# - proctitle: full command line +# - argc/a0/a1/a2: argument count and values +# - auid: original user (even through sudo) +# - cwd: directory where command was run +# +# View all audit keys available: +# sudo auditctl -l | grep -o 'key=[^ ]*' | sort -u +# +# Real-time monitoring: +# sudo ausearch -k terraform_command -ts recent -i | tail -f +# +# CloudWatch Logs query (after logs are shipped): +# Log group: /aws/ec2/terraformer +# Stream: audit/security +# Filter: terraform_command +# +# ============================================================ + +# ============================================================ +# TERRAFORM COMMAND EXECUTION - CRITICAL +# ============================================================ + +# Track all terraform command execution with full arguments +# This captures: who (auid), when (timestamp), what (command + args) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/terraform -k terraform_command +-a always,exit -F arch=b32 -S execve -F exe=/usr/bin/terraform -k terraform_command + +# Track terraform binary execution (simpler rule, backup) +-w /usr/bin/terraform -p x -k terraform_execution + +# Track OpenTofu if installed (open source fork) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/tofu -k tofu_command +-w /usr/bin/tofu -p x -k tofu_execution + +# ============================================================ +# TERRAFORM STATE FILES +# ============================================================ +# State is stored in S3 (terraform.tfstate) - audited by AWS CloudTrail, not here. +# CloudTrail logs: s3:GetObject, s3:PutObject on the state bucket. +# +# If you ever use local state files, uncomment: +# -w /var/lib/terraform/ -p rwa -k terraform_state +# -w /opt/terraform/ -p rwa -k terraform_state + +# ============================================================ +# AWS CREDENTIALS ACCESS +# ============================================================ + +# Track AWS credentials access (terraform typically uses AWS) +-w /root/.aws/ -p r -k aws_credentials_access +# Note: Cannot watch /home/*/.aws/ with -w rules (no wildcard support) + +# Track AWS environment variables file if used +-w /etc/environment -p r -k environment_access + +# ============================================================ +# TERRAFORM CONFIGURATION CHANGES +# ============================================================ + +# Track changes to terraform configurations in common locations +# Add project-specific paths as needed +-w /etc/terraform/ -p wa -k terraform_config + +# ============================================================ +# PRIVILEGED COMMAND EXECUTION +# ============================================================ + +# Track sudo usage (terraform often runs via sudo) +-a always,exit -F arch=b64 -S execve -F exe=/usr/bin/sudo -k sudo_usage + +# Track all commands run by non-system users +-a always,exit -F arch=b64 -S execve -F auid>=1000 -F auid!=4294967295 -k user_commands + +# ============================================================ +# GIT OPERATIONS (infrastructure as code) +# ============================================================ + +# Track git operations (terraform code changes) +-w /usr/bin/git -p x -k git_execution