Merge pull request #133 from NHSDigital/CCM-14510_FixTestAndTerraformMakTargets

CCM-14510 Fix Test and TF Make Targets

Author: aidenvaines-cgi

.github/actions/lint-terraform/action.yaml

@@ -7,15 +7,16 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: "Install Terraform binary"
+      shell: bash
+      run: |
+        asdf plugin add terraform || true
+        asdf install terraform || true
     - name: "Check Terraform format"
       shell: bash
       run: |
         check_only=true scripts/githooks/check-terraform-format.sh
     - name: "Validate Terraform"
       shell: bash
       run: |
-        stacks=${{ inputs.root-modules }}
-        for dir in $(find infrastructure/environments -maxdepth 1 -mindepth 1 -type d; echo ${stacks//,/$'\n'}); do
-          dir=$dir opts='-backend=false' make terraform-init
-          dir=$dir make terraform-validate
-        done
+        make terraform-validate-all

.github/workflows/stage-1-commit.yaml

@@ -148,6 +148,8 @@ jobs:
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v4
+      - name: "Setup ASDF"
+        uses: asdf-vm/actions/setup@1902764435ca0dd2f3388eea723a4f92a4eb8302
       - name: "Lint Terraform"
         uses: ./.github/actions/lint-terraform
   trivy-iac:

infrastructure/terraform/components/dnsroot/sns_topic_costs.tf

@@ -1,5 +1,5 @@
 resource "aws_sns_topic" "costs" {
-  name              = "${local.csi}-costs"
+  name = "${local.csi}-costs"
 }
 
 resource "aws_sns_topic_policy" "costs" {

scripts/config/pre-commit.yaml

@@ -13,6 +13,7 @@ repos:
       - id: mixed-line-ending
       - id: pretty-format-json
         args: ['--autofix']
+        exclude: '(^|/)package(-lock)?\.json$'
     # -   id: ...
   - repo: local
     hooks:

scripts/docker/examples/python/assets/hello_world/requirements.txt

@@ -8,5 +8,5 @@ MarkupSafe==2.1.3
 pip==23.3
 setuptools==78.1.1
 Werkzeug==3.0.6
-wheel==0.41.1
+wheel==0.46.2
 WTForms==3.0.1

scripts/githooks/check-terraform-format.sh

@@ -29,11 +29,11 @@ function main() {
 #   check_only=[do not format, run check only]
 function terraform-fmt() {
 
-  local opts=
   if is-arg-true "$check_only"; then
-    opts="-check"
+    make terraform-fmt-check
+  else
+    make terraform-fmt
   fi
-  opts=$opts make terraform-fmt
 }
 
 # ==============================================================================

scripts/githooks/check-todos.sh

@@ -120,7 +120,7 @@ function search_todos() {
 
     # If the file is excluded, skip it
     if [ "$skip" = false ] && [ -f "$file" ]; then
-      file_todos=$(grep -nHiE '\bTODO\b' "$file" || true)
+      file_todos=$(grep -nHiE '\bTODO(:| )' "$file" || true)
       [ -n "$file_todos" ] && todos+="$file_todos\n"
     fi
   done
@@ -136,7 +136,7 @@ function filter_todos_with_valid_jira_ticket() {
 
   while IFS= read -r line; do
     # Only lines with TODO but without a valid JIRA ticket
-    if grep -qnHiE '\bTODO\b' <<< "$line"; then
+    if grep -qnHiE '\bTODO(:| )' <<< "$line"; then
       if ! [[ "$line" =~ $jira_regex ]]; then
         todos_without_ticket+="$line\n"
       fi

scripts/init.mk

@@ -46,8 +46,8 @@ _install-dependency: # Install asdf dependency - mandatory: name=[listed in the
 	asdf install ${name} $(or ${version},)
 
 _install-dependencies: # Install all the dependencies listed in .tool-versions
-	for plugin in $$(grep ^[a-z] .tool-versions | sed 's/[[:space:]].*//'); do
-		make _install-dependency name="$${plugin}"
+	for plugin in $$(grep '^[a-z]' .tool-versions | cut -f1 -d' '); do \
+		$(MAKE) _install-dependency name=$${plugin}; \
 	done
 
 clean:: # Remove all generated and temporary files (common) @Operations

scripts/terraform/terraform.mk

@@ -1,110 +1,173 @@
-# This file is for you! Edit it to implement your own Terraform make targets.
+# Terraform Make Targets for TFScaffold
+# NHS Notify standard for production infrastructure
+# Requires infrastructure/terraform/bin/terraform.sh
 
 # ==============================================================================
-# Custom implementation - implementation of a make target should not exceed 5 lines of effective code.
-# In most cases there should be no need to modify the existing make targets.
-
-terraform-init: # Initialise Terraform - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform init command, default is none/empty] @Development
-	make _terraform cmd="init" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-terraform-plan: # Plan Terraform changes - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform plan command, default is none/empty] @Development
-	make _terraform cmd="plan" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-terraform-apply: # Apply Terraform changes - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform apply command, default is none/empty] @Development
-	make _terraform cmd="apply" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-terraform-destroy: # Destroy Terraform resources - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform destroy command, default is none/empty] @Development
-	make _terraform \
-		cmd="destroy" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-terraform-fmt: # Format Terraform files - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform fmt command, default is '-recursive'] @Quality
-	make _terraform cmd="fmt" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-terraform-validate: # Validate Terraform configuration - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform validate command, default is none/empty] @Quality
-	make _terraform cmd="validate" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-clean:: # Remove Terraform files (terraform) - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set] @Operations
-	make _terraform cmd="clean" \
-		dir=$(or ${terraform_dir}, ${dir}) \
-		opts=$(or ${terraform_opts}, ${opts})
-
-_terraform: # Terraform command wrapper - mandatory: cmd=[command to execute]; optional: dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], opts=[options to pass to the Terraform command, default is none/empty]
-	# 'TERRAFORM_STACK' is passed to the functions as environment variable
-	TERRAFORM_STACK=$(or ${TERRAFORM_STACK}, $(or ${terraform_stack}, $(or ${STACK}, ${stack})))
-	dir=$(or ${dir}, ${TERRAFORM_STACK})
-	. "scripts/terraform/terraform.lib.sh"; \
-	terraform-${cmd} # 'dir' and 'opts' are accessible by the function as environment variables, if set
+# TFScaffold Terraform Operations
+
+terraform-plan: # Plan Terraform changes - mandatory: component=[component_name], environment=[environment]; optional: project=[default: nhs], region=[default: eu-west-2], group=[default: dev], opts=[additional options] @Development
+	# Example: make terraform-plan component=mycomp environment=myenv group=mygroup
+	# Args: --project nhs --region eu-west-2 --component mycomp --environment myenv --group mygroup --action plan
+	make _terraform-scaffold action=plan \
+		component=$(component) \
+		environment=$(environment) \
+		project=$(or ${project}, nhs) \
+		region=$(or ${region}, eu-west-2) \
+		group=$(or ${group}, dev) \
+		opts=$(or ${opts}, )
+
+terraform-plan-destroy: # Plan Terraform destroy - mandatory: component=[component_name], environment=[environment]; optional: project, region, group, opts @Development
+	# Example: make terraform-plan-destroy component=mycomp environment=myenv group=mygroup
+	# Args: --project nhs --region eu-west-2 --component mycomp --environment myenv --group mygroup --action plan-destroy
+	make _terraform-scaffold action=plan-destroy \
+		component=$(component) \
+		environment=$(environment) \
+		project=$(or ${project}, nhs) \
+		region=$(or ${region}, eu-west-2) \
+		group=$(or ${group}, dev) \
+		opts=$(or ${opts}, )
+
+terraform-apply: # Apply Terraform changes - mandatory: component=[component_name], environment=[environment]; optional: project, region, group, build_id, opts @Development
+	# Example: make terraform-apply component=mycomp environment=myenv group=mygroup
+	# Args: --project nhs --region eu-west-2 --component mycomp --environment myenv --group mygroup --action apply
+	make _terraform-scaffold action=apply \
+		component=$(component) \
+		environment=$(environment) \
+		project=$(or ${project}, nhs) \
+		region=$(or ${region}, eu-west-2) \
+		group=$(or ${group}, dev) \
+		build_id=$(or ${build_id}, ) \
+		opts=$(or ${opts}, )
+
+terraform-destroy: # Destroy Terraform resources - mandatory: component=[component_name], environment=[environment]; optional: project, region, group, opts @Development
+	# Example: make terraform-destroy component=mycomp environment=myenv group=mygroup
+	# Args: --project nhs --region eu-west-2 --component mycomp --environment myenv --group mygroup --action destroy
+	make _terraform-scaffold action=destroy \
+		component=$(component) \
+		environment=$(environment) \
+		project=$(or ${project}, nhs) \
+		region=$(or ${region}, eu-west-2) \
+		group=$(or ${group}, dev) \
+		opts=$(or ${opts}, )
+
+terraform-output: # Get Terraform outputs - mandatory: component=[component_name], environment=[environment]; optional: project, region, group @Development
+	# Example: make terraform-output component=mycomp environment=myenv group=mygroup
+	# Args: --project nhs --region eu-west-2 --component mycomp --environment myenv --group mygroup --action output
+	make _terraform-scaffold action=output \
+		component=$(component) \
+		environment=$(environment) \
+		project=$(or ${project}, nhs) \
+		region=$(or ${region}, eu-west-2) \
+		group=$(or ${group}, dev)
+
+_terraform-scaffold: # Internal wrapper for terraform.sh - mandatory: action=[terraform action]; optional: component, environment, project, region, group, bootstrap, build_id, opts
+	cd infrastructure/terraform && \
+	if [ "$(bootstrap)" = "true" ]; then \
+		./bin/terraform.sh \
+			--bootstrap \
+			--project $(project) \
+			--region $(region) \
+			--group $(group) \
+			--action $(action) \
+			$(if $(opts),-- $(opts),); \
+	else \
+		./bin/terraform.sh \
+			--project $(project) \
+			--region $(region) \
+			--component $(component) \
+			--environment $(environment) \
+			--group $(group) \
+			$(if $(build_id),--build-id $(build_id),) \
+			--action $(action) \
+			$(if $(opts),-- $(opts),); \
+	fi
 
 # ==============================================================================
-# Quality checks - please DO NOT edit this section!
-
-terraform-shellscript-lint: # Lint all Terraform module shell scripts @Quality
-	for file in $$(find scripts/terraform -type f -name "*.sh"); do
-		file=$${file} scripts/shellscript-linter.sh
-	done
-
-terraform-sec: # TFSEC check against Terraform files - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform fmt command, default is '-recursive'] @Quality
-	tfsec infrastructure/terraform \
-		--force-all-dirs \
-		--exclude-downloaded-modules \
-		--config-file scripts/config/tfsec.yaml
-
-terraform-docs: # Terraform-docs check against Terraform files - optional: terraform_dir|dir=[path to a directory where the command will be executed, relative to the project's top-level directory, default is one of the module variables or the example directory, if not set], terraform_opts|opts=[options to pass to the Terraform fmt command, default is '-recursive'] @Quality
-	for dir in ./infrastructure/terraform/components/* ./infrastructure/terraform/modules/*; do \
+# Formatting and Validation
+
+terraform-fmt: # Format Terraform files in components/ and modules/ (excludes etc/) @Quality
+	# Example: make terraform-fmt
+	@cd infrastructure/terraform && \
+		for dir in components modules; do \
+			[ -d "$$dir" ] && terraform fmt -recursive "$$dir"; \
+		done
+
+terraform-fmt-check: # Check Terraform formatting in components/ and modules/ (excludes etc/) @Quality
+	# Example: make terraform-fmt-check
+	@cd infrastructure/terraform && \
+		for dir in components modules; do \
+			[ -d "$$dir" ] && terraform fmt -check -recursive "$$dir"; \
+		done
+
+terraform-validate: # Validate Terraform configuration - mandatory: component=[component_name] @Quality
+	# Example: make terraform-validate component=mycomp
+	# Note: Validation does not require environment/group as it checks syntax only
+	cd infrastructure/terraform/components/$(component) && \
+	terraform init -backend=false && \
+	terraform validate
+
+terraform-validate-all: # Validate all Terraform components @Quality
+	# Example: make terraform-validate-all
+	for dir in infrastructure/terraform/components/*; do \
 		if [ -d "$$dir" ]; then \
-			./scripts/terraform/terraform-docs.sh $$dir; \
-		fi \
+			echo "Validating $$(basename $$dir)..."; \
+			cd $$dir && \
+			terraform init -backend=false && \
+			terraform validate && \
+			cd - > /dev/null; \
+		fi; \
 	done
 
-# ==============================================================================
-# Module tests and examples - please DO NOT edit this section!
+terraform-sec: # Run Trivy IaC security scanning on Terraform code @Quality
+	# Example: make terraform-sec
+	./scripts/terraform/trivy-scan.sh --mode iac infrastructure/terraform
+
+terraform-docs: # Generate Terraform documentation - optional: component=[specific component, or all if omitted] @Quality
+	# Example: make terraform-docs component=mycomp
+	# Example: make terraform-docs (generates for all components)
+	@if [ -n "$(component)" ]; then \
+		./scripts/terraform/terraform-docs.sh infrastructure/terraform/components/$(component); \
+	else \
+		for dir in infrastructure/terraform/components/* infrastructure/terraform/modules/*; do \
+			if [ -d "$$dir" ]; then \
+				./scripts/terraform/terraform-docs.sh $$dir; \
+			fi; \
+		done; \
+	fi
 
-terraform-example-provision-aws-infrastructure: # Provision example of AWS infrastructure @ExamplesAndTests
-	make terraform-init
-	make terraform-plan opts="-out=terraform.tfplan"
-	make terraform-apply opts="-auto-approve terraform.tfplan"
-
-terraform-example-destroy-aws-infrastructure: # Destroy example of AWS infrastructure @ExamplesAndTests
-	make terraform-destroy opts="-auto-approve"
+# ==============================================================================
+# Cleanup
 
-terraform-example-clean: # Remove Terraform example files @ExamplesAndTests
-	dir=$(or ${dir}, ${TERRAFORM_STACK})
-	. "scripts/terraform/terraform.lib.sh"; \
-	terraform-clean
-	rm -f ${TERRAFORM_STACK}/.terraform.lock.hcl
+clean:: # Remove Terraform build artifacts and cache @Operations
+	# Example: make clean
+	rm -rf infrastructure/terraform/components/*/build
+	rm -rf infrastructure/terraform/components/*/.terraform
+	rm -rf infrastructure/terraform/components/*/.terraform.lock.hcl
+	rm -rf infrastructure/terraform/bootstrap/.terraform
+	rm -rf infrastructure/terraform/bootstrap/.terraform.lock.hcl
+	rm -rf infrastructure/terraform/plugin-cache/*
 
 # ==============================================================================
-# Configuration - please DO NOT edit this section!
+# Installation
 
-terraform-install: # Install Terraform @Installation
+terraform-install: # Install Terraform using asdf @Installation
+	# Example: make terraform-install
 	make _install-dependency name="terraform"
 
 # ==============================================================================
 
 ${VERBOSE}.SILENT: \
-	_terraform \
+	_terraform-scaffold \
 	clean \
 	terraform-apply \
 	terraform-destroy \
-	terraform-example-clean \
-	terraform-example-destroy-aws-infrastructure \
-	terraform-example-provision-aws-infrastructure \
-	terraform-fmt \
 	terraform-docs \
-	terraform-init \
+	terraform-fmt \
+	terraform-fmt-check \
 	terraform-install \
+	terraform-output \
 	terraform-plan \
-	terraform-shellscript-lint \
+	terraform-plan-destroy \
+	terraform-sec \
 	terraform-validate \
+	terraform-validate-all \

scripts/tests/test.mk

@@ -69,12 +69,14 @@ test: # Run all the test tasks @Testing
 		test-load
 
 _test:
-	set -e
-	script="./scripts/tests/${name}.sh"
-	if [ -e "$${script}" ]; then
-		exec $${script}
-	else
-		echo "make test-${name} not implemented: $${script} not found" >&2
+	set -e; \
+	script="./scripts/tests/${name}.sh"; \
+	if [ -e "$${script}" ]; then \
+		exec $${script}; \
+	else \
+		echo "test-${name}: Not currently implemented"; \
+		echo "Create $${script} to implement this test target"; \
+		exit 0; \
 	fi
 
 ${VERBOSE}.SILENT: \