Skip to content

Commit 11de9c6

Browse files
bmonkmanbmonkman
authored
feat: Added support for external-secrets, modified secret generation during apply step to use this new process (#203)
* feat: Added support for external-secrets, modified secret generation during apply step to use this new process * fix: Renamed secret templates from yml to json * fix: Add tf refresh before teardown, tear down application secrets too
1 parent 5b72481 commit 11de9c6

File tree

15 files changed

+193
-49
lines changed

15 files changed

+193
-49
lines changed

‎templates/Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,20 +80,25 @@ teardown-secrets:
8080
aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='cf-keypair' && Value=='$(PROJECT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
8181
aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='wg' && Value=='$(PROJECT)-$(ENVIRONMENT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
8282
aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='oathkeeper-jwks' && Value=='$(PROJECT)-$(ENVIRONMENT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
83+
aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='application-secret' && Value=='$(PROJECT)-$(ENVIRONMENT)-$(PROJECT)']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
84+
aws secretsmanager list-secrets --region $(AWS_DEFAULT_REGION) --query "SecretList[?Tags[?Key=='application-secret' && Value=='$(PROJECT)-$(ENVIRONMENT)-user-auth']].[Name] | [0][0]" | xargs aws secretsmanager delete-secret --region $(AWS_DEFAULT_REGION) --secret-id || echo "Secret already removed"
8385
aws iam list-role-policies --role-name $(PROJECT)-eks-cluster-creator --query "PolicyNames" | jq -r ".[]" | xargs -n1 aws iam delete-role-policy --role-name $(PROJECT)-eks-cluster-creator --policy-name
8486
aws iam list-attached-role-policies --role-name $(PROJECT)-eks-cluster-creator --query "AttachedPolicies[].PolicyArn" | jq -r ".[]" | xargs -n1 aws iam detach-role-policy --role-name $(PROJECT)-eks-cluster-creator --policy-arn
8587
aws iam delete-role --role-name $(PROJECT)-eks-cluster-creator
8688

8789
teardown-env:
8890
cd terraform/environments/$(ENVIRONMENT) && \
91+
terraform refresh && \
8992
terraform destroy
9093

9194
teardown-shared-env:
9295
cd terraform/environments/shared && \
96+
terraform refresh && \
9397
terraform destroy
9498

9599
teardown-k8s-utils:
96100
cd kubernetes/terraform/environments/$(ENVIRONMENT) && \
101+
terraform refresh && \
97102
terraform destroy
98103

99104
.PHONY: apply apply-remote-state apply-secrets apply-env apply-k8s-utils teardown-k8s-utils teardown-env teardown-shared-env teardown-secrets teardown-remote-state teardown-shared-remote-state

‎templates/kubernetes/terraform/environments/prod/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ module "kubernetes" {
8989
jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
9090
# This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
9191
user_auth_mail_from_address = "noreply@${local.domain_name}"
92-
cookie_sigining_secret_key = "${local.project}-${local.environment}-${local.random_seed}"
92+
cookie_signing_secret_key = "${local.project}-${local.environment}-${local.random_seed}"
9393
}
9494
## User auth: Kratos requires database and a secret (as: `user_auth[0].name`)
9595
## Oathkeeper requires a private key (as `user_auth[0].jwks_secret_name`)

‎templates/kubernetes/terraform/environments/stage/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ module "kubernetes" {
8888
jwks_secret_name = "${local.project}-${local.environment}-oathkeeper-jwks-${local.random_seed}"
8989
# This domain or address must be verified by the mail provider (Sendgrid, SES, etc.)
9090
user_auth_mail_from_address = "noreply@${local.domain_name}"
91-
cookie_sigining_secret_key = "${local.project}-${local.environment}-${local.random_seed}"
91+
cookie_signing_secret_key = "${local.project}-${local.environment}-${local.random_seed}"
9292
}
9393
## User auth: Kratos requires database and a secret (as: `user_auth[0].name`)
9494
## Oathkeeper requires a private key (as `user_auth[0].jwks_secret_name`)

‎templates/kubernetes/terraform/modules/kubernetes/external_secrets.tf‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
locals {
2+
external_secrets_namespace = "kube-system"
3+
4+
// Set / expose poller interval?
5+
external_secrets_helm_values = {
6+
nameOverride : "external-secrets"
7+
serviceMonitor : {
8+
enabled : (var.metrics_type == "prometheus")
9+
namespace : "metrics"
10+
}
11+
serviceAccount : {
12+
name : "external-secrets"
13+
annotations : {
14+
"eks.amazonaws.com/role-arn" : module.iam_assumable_role_external_secrets.this_iam_role_arn
15+
}
16+
}
17+
securityContext : {
18+
fsGroup : 65534
19+
}
20+
env : {
21+
AWS_REGION : var.region
22+
LOG_LEVEL : "warn" # use "info" to see all polling events
23+
# Each request to Secrets Manager has a small cost ($0.05 per 10,000 API calls) so a longer interval will reduce the number of calls but it will take longer to get updated secret values
24+
POLLER_INTERVAL_MILLISECONDS : 15000
25+
}
26+
}
27+
}
28+
29+
resource "helm_release" "external_secrets" {
30+
name = "external-secrets"
31+
repository = "https://external-secrets.github.io/kubernetes-external-secrets/"
32+
chart = "kubernetes-external-secrets"
33+
version = "7.2.1"
34+
namespace = local.external_secrets_namespace
35+
values = [jsonencode(local.external_secrets_helm_values)]
36+
}
37+
38+
# Create a role using oidc to map service accounts
39+
module "iam_assumable_role_external_secrets" {
40+
source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc"
41+
version = "~> v3.15.0"
42+
create_role = true
43+
role_name = "${var.project}-k8s-${var.environment}-external-secrets"
44+
provider_url = replace(data.aws_eks_cluster.cluster.identity.0.oidc.0.issuer, "https://", "")
45+
role_policy_arns = [aws_iam_policy.external_secrets.arn]
46+
oidc_fully_qualified_subjects = ["system:serviceaccount:${local.external_secrets_namespace}:external-secrets"]
47+
}
48+
49+
resource "aws_iam_policy" "external_secrets" {
50+
name_prefix = "kubernetes-external-secrets"
51+
description = "Kubernetes External Secrets Policy"
52+
policy = data.aws_iam_policy_document.external_secrets_policy_doc.json
53+
}
54+
55+
data "aws_iam_policy_document" "external_secrets_policy_doc" {
56+
statement {
57+
effect = "Allow"
58+
resources = ["arn:aws:secretsmanager:${var.region}:*:secret:${var.project}/kubernetes/${var.environment}/*"]
59+
60+
actions = [
61+
"secretsmanager:GetResourcePolicy",
62+
"secretsmanager:GetSecretValue",
63+
"secretsmanager:DescribeSecret",
64+
"secretsmanager:ListSecretVersionIds",
65+
]
66+
}
67+
}

‎templates/kubernetes/terraform/modules/kubernetes/user_auth.tf‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,23 @@
1+
locals {
2+
# This secret is created by the /scripts/create-db-user.sh script and contains environment variables that will be pulled into a k8s secret automatically by external-secrets
3+
secrets_manager_secret_name = "${var.project}/kubernetes/${var.environment}/user-auth"
4+
}
5+
6+
7+
## Get generated JWKS content from secret
8+
data "aws_secretsmanager_secret" "jwks_content" {
9+
count = length(var.user_auth)
10+
name = var.user_auth[count.index].jwks_secret_name
11+
}
12+
data "aws_secretsmanager_secret_version" "jwks_content" {
13+
count = length(data.aws_secretsmanager_secret.jwks_content)
14+
secret_id = data.aws_secretsmanager_secret.jwks_content[count.index].id
15+
}
16+
117
module "user_auth" {
218
count = length(var.user_auth)
319
source = "commitdev/zero/aws//modules/user_auth"
4-
version = "0.1.21"
20+
version = "0.3.6"
521

622
name = var.user_auth[count.index].name
723
auth_namespace = var.user_auth[count.index].auth_namespace
@@ -11,7 +27,10 @@ module "user_auth" {
1127
backend_service_domain = var.user_auth[count.index].backend_service_domain
1228
user_auth_mail_from_address = var.user_auth[count.index].user_auth_mail_from_address
1329
whitelisted_return_urls = var.user_auth[count.index].whitelisted_return_urls
14-
jwks_secret_name = var.user_auth[count.index].jwks_secret_name
15-
cookie_sigining_secret_key = var.user_auth[count.index].cookie_sigining_secret_key
16-
k8s_local_exec_context = local.k8s_exec_context
30+
jwks_content = data.aws_secretsmanager_secret_version.jwks_content[count.index].secret_string
31+
cookie_signing_secret_key = var.user_auth[count.index].cookie_signing_secret_key
32+
kubectl_extra_args = local.k8s_exec_context
33+
external_secret_name = local.secrets_manager_secret_name
34+
35+
depends_on = [helm_release.external_secrets]
1736
}

‎templates/kubernetes/terraform/modules/kubernetes/variables.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ variable "user_auth" {
144144
jwks_secret_name = string
145145
user_auth_mail_from_address = string
146146
whitelisted_return_urls = list(string)
147-
cookie_sigining_secret_key = string
147+
cookie_signing_secret_key = string
148148
}))
149149
}
150150

‎templates/scripts/create-db-user.sh‎

Lines changed: 13 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,25 @@ usage () {
99
# check parameters
1010
# REGION - AWS region to use
1111
# SEED - Random seed that is part of the name of the AWS secret containing the db master password
12-
# PROJECT_NAME - Name of the project and the k8s namespace containing the database service
12+
# PROJECT_NAME - Name of the project
1313
# ENVIRONMENT - stage or prod
14-
# NAMESPACE - The target k8s namespace to create and create a secret in
14+
# NAMESPACE - The target k8s namespace to create a secret in
1515
# DATABASE_TYPE - The type of database - mysql, postgres
1616
# DATABASE_NAME - The name of the database(s) to create in the database server
1717
# USER_NAME - The name of the user to create and grant access to the database specified above
1818
# USER_PASSWORD - The password of the user to create and grant access to the database specified above (optional)
19-
# SECRET_NAME - The secret of the database user to get password
19+
# SECRET_NAME - The suffix name of the secret created in AWS Secret Manager that will contain the created credentials
2020
# CREATE_SECRET - A template file to render to create a secret (optional)
21-
# CREATE_DB_POD - A template file to render to create a db pod for troubleshooting (optional)
2221
([[ -z "${REGION}" ]] || \
2322
[[ -z "${SEED}" ]] || \
2423
[[ -z "${PROJECT_NAME}" ]] || \
2524
[[ -z "${ENVIRONMENT}" ]] || \
2625
[[ -z "${NAMESPACE}" ]] || \
26+
[[ -z "${SECRET_NAME}" ]] || \
2727
[[ -z "${DATABASE_TYPE}" ]] || \
2828
[[ -z "${DATABASE_NAME}" ]] || \
29-
[[ -z "${SECRET_NAME}" ]] || \
3029
[[ -z "${USER_NAME}" ]] ) && \
31-
echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMESPACE, DATABASE_TYPE, DATABASE_NAME, USER_NAME) are not set properly." && usage
30+
echo "Some environment variables (REGION, SEED, PROJECT_NAME, ENVIRONMENT, NAMESPACE, SECRET_NAME, DATABASE_TYPE, DATABASE_NAME, USER_NAME) are not set properly." && usage
3231

3332
# docker image with postgres + mysql clients
3433
DOCKER_IMAGE_TAG=commitdev/zero-k8s-utilities:0.0.3
@@ -46,6 +45,7 @@ MASTER_RDS_PASSWORD=$(aws secretsmanager get-secret-value --region=${REGION} --s
4645
## get application user/pass
4746
DB_APP_USERNAME=$(echo "${USER_NAME}" | tr -dc 'A-Za-z0-9')
4847
DB_APP_PASSWORD=${USER_PASSWORD:-$(LC_ALL=C tr -dc 'A-Za-z0-9' < /dev/urandom | base64 | head -c 24)}
48+
JOB_ID=$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
4949

5050
# get correct dsn string for db type
5151
if [[ "${DB_TYPE}" == "postgres" ]]; then
@@ -54,28 +54,24 @@ elif [[ "${DB_TYPE}" == "mysql" ]]; then
5454
DB_ENDPOINT_FOR_DSN="tcp(${DB_ENDPOINT})"
5555
fi
5656

57-
# fill in env-vars to db user creation manifest
58-
JOB_ID=$(LC_ALL=C tr -dc 'a-z0-9' < /dev/urandom | head -c 8)
59-
eval "echo \"$(cat ./db-ops/job-create-db-${DATABASE_TYPE}.yml.tpl)\"" > ./k8s-job-create-db.yml
60-
[[ -z "${CREATE_SECRET}" ]] || eval "echo \"$(cat ./db-ops/${CREATE_SECRET})\"" >> ./k8s-job-create-db.yml
6157
# the manifest creates these things
6258
# 1. Namespaces: db-ops, $NAMESPACE
6359
# 2. Secret in db-ops: db-create-users (with master password, and a .sql file
6460
# 3. Job in db-ops: db-create-users (runs the .sql file against the RDS given master_password from env)
65-
# 4. Secret in $NAMESPACE namespace with DB_USERNAME / DB_PASSWORD
6661

67-
# execution
68-
kubectl apply -f ./k8s-job-create-db.yml
69-
rm -f ./k8s-job-create-db.yml
62+
# Run the job in the kubernetes cluster that will create the database user
63+
eval "echo \"$(cat ./db-ops/job-create-db-${DATABASE_TYPE}.yml.tpl)\"" | kubectl apply -f -
64+
65+
# Create a secret in AWS Secrets Manager. The contents of this secret will be automatically pulled into a kubernetes secret by external-secrets
66+
[[ -z "${CREATE_SECRET}" ]] || aws secretsmanager create-secret --name "${PROJECT_NAME}/kubernetes/${ENVIRONMENT}/${SECRET_NAME}" --description "Application secrets" --tags "[{\"Key\":\"application-secret\",\"Value\":\"${PROJECT}-${ENVIRONMENT}-${SECRET_NAME}\"}]" --secret-string "$(eval "echo \"$(cat ./db-ops/${CREATE_SECRET})\"")"
7067

71-
# clean up
72-
## Deleting the entire db-ops namespace, leaving ONLY application-namespace's secret behind
68+
## Delete the entire db-ops namespace
7369
kubectl -n db-ops wait --for=condition=complete --timeout=10s job db-create-users-$NAMESPACE-${JOB_ID}
7470
if [ $? -eq 0 ]
7571
then
7672
kubectl delete namespace db-ops
7773
else
7874
echo "Failed to create application database user, please see 'kubectl logs -n db-ops -l job-name=db-create-users-$NAMESPACE-${JOB_ID}'"
79-
kubectl delete secret -n db-ops ${SECRET_NAME}
75+
kubectl delete secret -n db-ops db-create-users
8076
fi
8177

‎templates/scripts/db-ops/secret-application.json.tpl‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
\"DATABASE_USERNAME\":\"$DB_APP_USERNAME\",
3+
\"DATABASE_PASSWORD\":\"$DB_APP_PASSWORD\"
4+
}

‎templates/scripts/db-ops/secret-application.yml.tpl‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎templates/scripts/db-ops/secret-user-auth.json.tpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
\"dsn\":\"$DB_TYPE://$DB_APP_USERNAME:$DB_APP_PASSWORD@$DB_ENDPOINT_FOR_DSN/$DB_NAME\",
3+
\"secretsCookie\":\"cookie-secret-$PROJECT_NAME-$SEED\",
4+
\"secretsDefault\":\"default-secret-$PROJECT_NAME-$SEED\",
5+
\"smtpConnectionURI\":\"$SMTP_URI\"
6+
}

0 commit comments

Comments
 (0)