diff --git a/.gitignore b/.gitignore index 75f6d4f..372eee1 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,5 @@ cython_debug/ # Secrets secrets/ !secrets/.keep_me + +.DS_Store diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..a27c73c --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,44 @@ +# Kubernetes Deployment + +The `base` directory contains Kubernetes manifest files to deploy the Clair Berlin stack to a Kubernetes cluster. + +## Environment and Configuration Management + +We use [Kustomize](https://kustomize.io/) to configure the stack for different environments. An environment's configuration consists of a set of environment variables used to generate a config map called `clair-config-map` and a set of password files used to generate service-specific secrets. The secret files in `base/secrets` do not contain any real passwords. You will have to create a [Kustomize overlay](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustomization/#bases-and-overlays) to override them. + +You can use the following shell script to generate the skeleton of a new environment (set ENV_NAME accordingly): + +```shell +ENV_NAME=my-env +ENV_DIR=environmemnts/$ENV_NAME +mkdir -p $ENV_DIR +cp -R base/config.env base/secrets environments/dev/kustomization.yaml $ENV_DIR +``` + +After that, edit `$ENV_DIR/config.env` and the files in `$ENV_DIR/secrets` to adapt the evironment's configuration. + +## cert-manager configuration + +The [SysEleven environment](environments/syseleven) contains an example how a cluster issuer and its corresponding annotation can be added to the Ingress object of the base layer. + +## Deployment + +To deploy an environment do the following: + +1) activate the target cluster's context using + +```shell +kubectl config use-context $ENV_CONTEXT +``` + +2) apply the kustomized manifest files + +```shell +kubectl apply -k $ENV_DIR +``` + +To delete the stack call + +```shell +kubectl delete -k $ENV_DIR +``` diff --git a/k8s/base/00-clair-berlin-namespace.yaml b/k8s/base/00-clair-berlin-namespace.yaml new file mode 100644 index 0000000..5ab394b --- /dev/null +++ b/k8s/base/00-clair-berlin-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: clair-berlin diff --git a/k8s/base/11-db-pvc.yaml b/k8s/base/11-db-pvc.yaml new file mode 100644 index 0000000..18a3da4 --- /dev/null +++ b/k8s/base/11-db-pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: db-data + namespace: clair-berlin +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi \ No newline at end of file diff --git a/k8s/base/12-db-service.yaml b/k8s/base/12-db-service.yaml new file mode 100644 index 0000000..d46a179 --- /dev/null +++ b/k8s/base/12-db-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: db + namespace: clair-berlin +spec: + selector: + app: db + ports: + - port: 5432 diff --git a/k8s/base/13-db-deployment.yaml b/k8s/base/13-db-deployment.yaml new file mode 100644 index 0000000..10a31bd --- /dev/null +++ b/k8s/base/13-db-deployment.yaml @@ -0,0 +1,47 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: db + name: db + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: db + strategy: + type: Recreate + template: + metadata: + labels: + app: db + spec: + containers: + - env: + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_DATABASE + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: db-secret + key: sql-password.txt + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_USER + image: postgres:13.1 + name: db + resources: {} + volumeMounts: + - mountPath: /var/lib/postgresql + name: db-data + restartPolicy: Always + volumes: + - name: db-data + persistentVolumeClaim: + claimName: db-data diff --git a/k8s/base/22-redis-service.yaml b/k8s/base/22-redis-service.yaml new file mode 100644 index 0000000..0ccd063 --- /dev/null +++ b/k8s/base/22-redis-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: clair-berlin +spec: + selector: + app: redis + ports: + - port: 6379 \ No newline at end of file diff --git a/k8s/base/23-redis-deployment.yaml b/k8s/base/23-redis-deployment.yaml new file mode 100644 index 0000000..5074345 --- /dev/null +++ b/k8s/base/23-redis-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: redis + name: redis + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: redis + strategy: {} + template: + metadata: + labels: + app: redis + spec: + containers: + - image: redis:6.0.9 + name: redis + resources: {} + restartPolicy: Always diff --git a/k8s/base/32-ingestair-service.yaml b/k8s/base/32-ingestair-service.yaml new file mode 100644 index 0000000..c19bfa0 --- /dev/null +++ b/k8s/base/32-ingestair-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: ingestair + namespace: clair-berlin +spec: + selector: + app: ingestair + ports: + - port: 8888 diff --git a/k8s/base/33-ingestair-deployment.yaml b/k8s/base/33-ingestair-deployment.yaml new file mode 100644 index 0000000..e2d8d02 --- /dev/null +++ b/k8s/base/33-ingestair-deployment.yaml @@ -0,0 +1,125 @@ + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingestair + name: ingestair + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: ingestair + strategy: {} + template: + metadata: + labels: + app: ingestair + spec: + containers: + - args: + - python + - manage.py + - runserver + - 0.0.0.0:8888 + env: + - name: SECRET_KEY_FILE + value: "/var/secrets/ingestair-secret/ingestair-secret-key.txt" + - name: SENTRY + value: "0" + - name: SENTRY_URL_FILE + value: "/var/secrets/ingestair-secret/sentry-url.txt" + - name: DEBUG + value: "0" + - name: DJANGO_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DJANGO_LOG_LEVEL + - name: DJANGO_DB_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DJANGO_DB_LOG_LEVEL + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: LOG_LEVEL + - name: SQL_ENGINE + value: "django.db.backends.postgresql" + - name: SQL_HOST + value: "db" + - name: SQL_PORT + value: "5432" + - name: SQL_DATABASE + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_DATABASE + - name: SQL_USER + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_USER + - name: SQL_PASSWORD_FILE + value: "/var/secrets/ingestair-secret/sql-password.txt" + # XXX why doesn't this work? + # - name: SQL_PASSWORD + # valueFrom: + # secretKeyRef: + # name: sql-password + # key: sql-password + - name: DATABASE + value: "postgresql" + - name: DB_MIGRATE + valueFrom: + configMapKeyRef: + name: clair-config-map + key: INGESTAIR_DB_MIGRATE + - name: COLLECT_STATIC_FILES + value: "false" + - name: NODE_FIDELITY + value: "1" + - name: DJANGO_ALLOWED_HOSTS + value: " ingestair localhost 127.0.0.1 [::1]" + - name: EMAIL_HOST + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_HOST + - name: EMAIL_PORT + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_PORT + - name: EMAIL_HOST_USER + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_HOST_USER + - name: EMAIL_HOST_PASSWORD_FILE + value: "/var/secrets/ingestair-secret/smtp-password.txt" + - name: EMAIL_USE_TLS + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_USE_TLS + - name: DEFAULT_FROM_EMAIL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DEFAULT_FROM_EMAIL + image: clairberlin/managair:0.6.5 + name: ingestair-server + resources: {} + volumeMounts: + - mountPath: "/var/secrets/ingestair-secret" + name: ingestair-secret + readOnly: true + restartPolicy: Always + volumes: + - name: ingestair-secret + secret: + secretName: ingestair-secret diff --git a/k8s/base/42-managair-service.yaml b/k8s/base/42-managair-service.yaml new file mode 100644 index 0000000..0405aae --- /dev/null +++ b/k8s/base/42-managair-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: managair-server + namespace: clair-berlin +spec: + selector: + app: managair-server + ports: + - port: 8888 \ No newline at end of file diff --git a/k8s/base/43-managair-deployment.yaml b/k8s/base/43-managair-deployment.yaml new file mode 100644 index 0000000..eee56bc --- /dev/null +++ b/k8s/base/43-managair-deployment.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: managair-server + name: managair-server + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: managair-server + strategy: {} + template: + metadata: + labels: + app: managair-server + spec: + containers: + - args: + - python + - manage.py + - runserver + - 0.0.0.0:8888 + env: + - name: SECRET_KEY_FILE + value: "/var/secrets/managair-secret/managair-secret-key.txt" + - name: SENTRY + value: "0" + - name: SENTRY_URL_FILE + value: "/var/secrets/managair-secret/sentry-url.txt" + - name: DEBUG + value: "0" + - name: DJANGO_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DJANGO_LOG_LEVEL + - name: DJANGO_DB_LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DJANGO_DB_LOG_LEVEL + - name: LOG_LEVEL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: LOG_LEVEL + - name: SQL_ENGINE + value: "django.db.backends.postgresql" + - name: SQL_HOST + value: "db" + - name: SQL_PORT + value: "5432" + - name: SQL_DATABASE + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_DATABASE + - name: SQL_USER + valueFrom: + configMapKeyRef: + name: clair-config-map + key: SQL_USER + - name: SQL_PASSWORD_FILE + value: "/var/secrets/managair-secret/sql-password.txt" + # XXX why doesn't this work? + # - name: SQL_PASSWORD + # valueFrom: + # secretKeyRef: + # name: sql-password + # key: sql-password + - name: DATABASE + value: "postgresql" + - name: DB_MIGRATE + valueFrom: + configMapKeyRef: + name: clair-config-map + key: MANAGAIR_DB_MIGRATE + - name: COLLECT_STATIC_FILES + valueFrom: + configMapKeyRef: + name: clair-config-map + key: MANAGAIR_COLLECT_STATIC_FILES + - name: NODE_FIDELITY + value: "1" + - name: CLAIR_DOMAIN + valueFrom: + configMapKeyRef: + name: clair-config-map + key: CLAIR_DOMAIN + - name: DJANGO_ALLOWED_HOSTS + value: " $(CLAIR_DOMAIN) localhost 127.0.0.1 [::1]" + - name: EMAIL_HOST + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_HOST + - name: EMAIL_PORT + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_PORT + - name: EMAIL_HOST_USER + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_HOST_USER + - name: EMAIL_HOST_PASSWORD_FILE + value: "/var/secrets/managair-secret/smtp-password.txt" + - name: EMAIL_USE_TLS + valueFrom: + configMapKeyRef: + name: clair-config-map + key: EMAIL_USE_TLS + - name: DEFAULT_FROM_EMAIL + valueFrom: + configMapKeyRef: + name: clair-config-map + key: DEFAULT_FROM_EMAIL + image: clairberlin/managair:0.6.5 + name: managair-server + resources: {} + volumeMounts: + - mountPath: "/var/secrets/managair-secret" + name: managair-secret + readOnly: true + restartPolicy: Always + volumes: + - name: managair-secret + secret: + secretName: managair-secret diff --git a/k8s/base/53-clairchen-forwarder-v2-deployment.yaml b/k8s/base/53-clairchen-forwarder-v2-deployment.yaml new file mode 100644 index 0000000..33a4770 --- /dev/null +++ b/k8s/base/53-clairchen-forwarder-v2-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: clairchen-forwarder-v2 + name: clairchen-forwarder-v2 + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: clairchen-forwarder-v2 + strategy: {} + template: + metadata: + labels: + app: clairchen-forwarder-v2 + spec: + containers: + - env: + - name: CLAIR_API_ROOT + value: "http://ingestair:8888/ingest/v1/" + - name: CLAIR_MODE + value: "clairchen-forward" + - name: CLAIR_TTN_ACCESS_KEY_FILE + value: "/var/secrets/clairchen-forwarder-v2-secret/clairchen-forwarder-v2-access-key.txt" + - name: CLAIR_TTN_APP_ID + value: "clairberlinproto" + - name: CLAIR_TTN_STACK + value: "ttn-v2" + image: clairberlin/clairttn:6 + livenessProbe: + exec: + command: + - /opt/clairttn/healthcheck.sh + initialDelaySeconds: 10 + periodSeconds: 5 + name: clairchen-forwarder-v2 + resources: {} + volumeMounts: + - mountPath: "/var/secrets/clairchen-forwarder-v2-secret" + name: clairchen-forwarder-v2-secret + readOnly: true + restartPolicy: Always + volumes: + - name: clairchen-forwarder-v2-secret + secret: + secretName: clairchen-forwarder-v2-secret diff --git a/k8s/base/54-ers-forwarder-v2-deployment.yaml b/k8s/base/54-ers-forwarder-v2-deployment.yaml new file mode 100644 index 0000000..bc79319 --- /dev/null +++ b/k8s/base/54-ers-forwarder-v2-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ers-forwarder-v2 + name: ers-forwarder-v2 + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: ers-forwarder-v2 + strategy: {} + template: + metadata: + labels: + app: ers-forwarder-v2 + spec: + containers: + - env: + - name: CLAIR_API_ROOT + value: "http://ingestair:8888/ingest/v1/" + - name: CLAIR_MODE + value: "ers-forward" + - name: CLAIR_TTN_ACCESS_KEY_FILE + value: "/var/secrets/ers-forwarder-v2-secret/ers-forwarder-v2-access-key.txt" + - name: CLAIR_TTN_APP_ID + value: "clair-berlin-ers-co2" + - name: CLAIR_TTN_STACK + value: "ttn-v2" + image: clairberlin/clairttn:6 + livenessProbe: + exec: + command: + - /opt/clairttn/healthcheck.sh + initialDelaySeconds: 10 + periodSeconds: 5 + name: ers-forwarder-v2 + resources: {} + volumeMounts: + - mountPath: "/var/secrets/ers-forwarder-v2-secret" + name: ers-forwarder-v2-secret + readOnly: true + restartPolicy: Always + volumes: + - name: ers-forwarder-v2-secret + secret: + secretName: ers-forwarder-v2-secret diff --git a/k8s/base/55-clairchen-forwarder-v3-deployment.yaml b/k8s/base/55-clairchen-forwarder-v3-deployment.yaml new file mode 100644 index 0000000..8be76fa --- /dev/null +++ b/k8s/base/55-clairchen-forwarder-v3-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: clairchen-forwarder-v3 + name: clairchen-forwarder-v3 + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: clairchen-forwarder-v3 + strategy: {} + template: + metadata: + labels: + app: clairchen-forwarder-v3 + spec: + containers: + - env: + - name: CLAIR_API_ROOT + value: "http://ingestair:8888/ingest/v1/" + - name: CLAIR_MODE + value: "clairchen-forward" + - name: CLAIR_TTN_ACCESS_KEY_FILE + value: "/var/secrets/clairchen-forwarder-v3-secret/clairchen-forwarder-v3-access-key.txt" + - name: CLAIR_TTN_APP_ID + value: "clairchen-test" + - name: CLAIR_TTN_STACK + value: "ttn-v3" + image: clairberlin/clairttn:6 + livenessProbe: + exec: + command: + - /opt/clairttn/healthcheck.sh + initialDelaySeconds: 10 + periodSeconds: 5 + name: clairchen-forwarder-v3 + resources: {} + volumeMounts: + - mountPath: "/var/secrets/clairchen-forwarder-v3-secret" + name: clairchen-forwarder-v3-secret + readOnly: true + restartPolicy: Always + volumes: + - name: clairchen-forwarder-v3-secret + secret: + secretName: clairchen-forwarder-v3-secret diff --git a/k8s/base/56-ers-forwarder-v3-deployment.yaml b/k8s/base/56-ers-forwarder-v3-deployment.yaml new file mode 100644 index 0000000..38b02be --- /dev/null +++ b/k8s/base/56-ers-forwarder-v3-deployment.yaml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ers-forwarder-v3 + name: ers-forwarder-v3 + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: ers-forwarder-v3 + strategy: {} + template: + metadata: + labels: + app: ers-forwarder-v3 + spec: + containers: + - env: + - name: CLAIR_API_ROOT + value: "http://ingestair:8888/ingest/v1/" + - name: CLAIR_MODE + value: "ers-forward" + - name: CLAIR_TTN_ACCESS_KEY_FILE + value: "/var/secrets/ers-forwarder-v3-secret/ers-forwarder-v3-access-key.txt" + - name: CLAIR_TTN_APP_ID + value: "elsys-ers-co2" + - name: CLAIR_TTN_STACK + value: "ttn-v3" + image: clairberlin/clairttn:6 + livenessProbe: + exec: + command: + - /opt/clairttn/healthcheck.sh + initialDelaySeconds: 10 + periodSeconds: 5 + name: ers-forwarder-v3 + resources: {} + volumeMounts: + - mountPath: "/var/secrets/ers-forwarder-v3-secret" + name: ers-forwarder-v3-secret + readOnly: true + restartPolicy: Always + volumes: + - name: ers-forwarder-v3-secret + secret: + secretName: ers-forwarder-v3-secret diff --git a/k8s/base/72-static-frontend-service.yaml b/k8s/base/72-static-frontend-service.yaml new file mode 100644 index 0000000..2624bfc --- /dev/null +++ b/k8s/base/72-static-frontend-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: static-frontend + namespace: clair-berlin +spec: + selector: + app: static-frontend + ports: + - port: 80 \ No newline at end of file diff --git a/k8s/base/73-static-frontend-deployment.yaml b/k8s/base/73-static-frontend-deployment.yaml new file mode 100644 index 0000000..d498580 --- /dev/null +++ b/k8s/base/73-static-frontend-deployment.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: static-frontend + name: static-frontend + namespace: clair-berlin +spec: + replicas: 1 + selector: + matchLabels: + app: static-frontend + strategy: {} + template: + metadata: + labels: + app: static-frontend + spec: + containers: + - image: clairberlin/website:15 + name: static-frontend + resources: {} + restartPolicy: Always diff --git a/k8s/base/99-clair-berlin-ingress.yaml b/k8s/base/99-clair-berlin-ingress.yaml new file mode 100644 index 0000000..ab20c57 --- /dev/null +++ b/k8s/base/99-clair-berlin-ingress.yaml @@ -0,0 +1,47 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: clair-berlin + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: static-frontend + port: + number: 80 + - path: /admin + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /static + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /accounts + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /api + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + diff --git a/k8s/base/config.env b/k8s/base/config.env new file mode 100644 index 0000000..93b0dc1 --- /dev/null +++ b/k8s/base/config.env @@ -0,0 +1,24 @@ +CLAIR_DOMAIN=clair-berlin.de + +DEBUG=0 +SENTRY=1 +DEBUG_TOOLBAR=0 + +DJANGO_LOG_LEVEL=WARNING +DJANGO_DB_LOG_LEVEL=WARNING +LOG_LEVEL=WARNING + +MANAGAIR_DB_MIGRATE=true +MANAGAIR_COLLECT_STATIC_FILES=true + +INGESTAIR_DB_MIGRATE=false + +SQL_DATABASE=managairdb_dev +SQL_USER=managair_dev +SQL_PASSWORD=postgres + +EMAIL_HOST=mx2ed5.netcup.net +EMAIL_PORT=587 +EMAIL_HOST_USER=clair-sender@clair-berlin.de +EMAIL_USE_TLS=True +DEFAULT_FROM_EMAIL=kontakt@clair-berlin.de diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml new file mode 100644 index 0000000..20b9072 --- /dev/null +++ b/k8s/base/kustomization.yaml @@ -0,0 +1,59 @@ +resources: + - 00-clair-berlin-namespace.yaml + - 11-db-pvc.yaml + - 12-db-service.yaml + - 13-db-deployment.yaml + - 22-redis-service.yaml + - 23-redis-deployment.yaml + - 32-ingestair-service.yaml + - 33-ingestair-deployment.yaml + - 42-managair-service.yaml + - 43-managair-deployment.yaml + - 53-clairchen-forwarder-v2-deployment.yaml + - 54-ers-forwarder-v2-deployment.yaml + - 55-clairchen-forwarder-v3-deployment.yaml + - 56-ers-forwarder-v3-deployment.yaml + - 72-static-frontend-service.yaml + - 73-static-frontend-deployment.yaml + - 99-clair-berlin-ingress.yaml + +configMapGenerator: + - name: clair-config-map + namespace: clair-berlin + envs: + - config.env + +secretGenerator: + - name: db-secret + namespace: clair-berlin + files: + - secrets/sql-password.txt + - name: managair-secret + namespace: clair-berlin + files: + - secrets/managair-secret-key.txt + - secrets/sql-password.txt + - secrets/sentry-url.txt + - secrets/smtp-password.txt + - name: ingestair-secret + namespace: clair-berlin + files: + - secrets/ingestair-secret-key.txt + - secrets/smtp-password.txt + - secrets/sql-password.txt + - name: clairchen-forwarder-v2-secret + namespace: clair-berlin + files: + - secrets/clairchen-forwarder-v2-access-key.txt + - name: ers-forwarder-v2-secret + namespace: clair-berlin + files: + - secrets/ers-forwarder-v3-access-key.txt + - name: clairchen-forwarder-v3-secret + namespace: clair-berlin + files: + - secrets/clairchen-forwarder-v3-access-key.txt + - name: ers-forwarder-v3-secret + namespace: clair-berlin + files: + - secrets/ers-forwarder-v3-access-key.txt diff --git a/k8s/base/secrets/clairchen-forwarder-v2-access-key.txt b/k8s/base/secrets/clairchen-forwarder-v2-access-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/clairchen-forwarder-v2-access-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/clairchen-forwarder-v3-access-key.txt b/k8s/base/secrets/clairchen-forwarder-v3-access-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/clairchen-forwarder-v3-access-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/ers-forwarder-v2-access-key.txt b/k8s/base/secrets/ers-forwarder-v2-access-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/ers-forwarder-v2-access-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/ers-forwarder-v3-access-key.txt b/k8s/base/secrets/ers-forwarder-v3-access-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/ers-forwarder-v3-access-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/ingestair-secret-key.txt b/k8s/base/secrets/ingestair-secret-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/ingestair-secret-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/managair-secret-key.txt b/k8s/base/secrets/managair-secret-key.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/managair-secret-key.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/sentry-url.txt b/k8s/base/secrets/sentry-url.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/sentry-url.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/smtp-password.txt b/k8s/base/secrets/smtp-password.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/smtp-password.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/base/secrets/sql-password.txt b/k8s/base/secrets/sql-password.txt new file mode 100644 index 0000000..1333ed7 --- /dev/null +++ b/k8s/base/secrets/sql-password.txt @@ -0,0 +1 @@ +TODO diff --git a/k8s/environments/dev/config.env b/k8s/environments/dev/config.env new file mode 100644 index 0000000..aa4ab48 --- /dev/null +++ b/k8s/environments/dev/config.env @@ -0,0 +1,24 @@ +CLAIR_DOMAIN=localhost + +DEBUG=1 +SENTRY=0 +DEBUG_TOOLBAR=1 + +DJANGO_LOG_LEVEL=DEBUG +DJANGO_DB_LOG_LEVEL=DEBUG +LOG_LEVEL=DEBUG + +MANAGAIR_DB_MIGRATE=true +MANAGAIR_COLLECT_STATIC_FILES=true + +INGESTAIR_DB_MIGRATE=false + +SQL_DATABASE=managairdb_dev +SQL_USER=managair_dev +SQL_PASSWORD=postgres + +EMAIL_HOST=mx2ed5.netcup.net +EMAIL_PORT=587 +EMAIL_HOST_USER=clair-sender@clair-berlin.de +EMAIL_USE_TLS=True +DEFAULT_FROM_EMAIL=kontakt@clair-berlin.de diff --git a/k8s/environments/dev/kustomization.yaml b/k8s/environments/dev/kustomization.yaml new file mode 100644 index 0000000..c638bcb --- /dev/null +++ b/k8s/environments/dev/kustomization.yaml @@ -0,0 +1,51 @@ +resources: + - ../../base + +configMapGenerator: + - name: clair-config-map + namespace: clair-berlin + behavior: replace + envs: + - config.env + +secretGenerator: + - name: db-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/sql-password.txt + - name: managair-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/managair-secret-key.txt + - secrets/sql-password.txt + - secrets/sentry-url.txt + - secrets/smtp-password.txt + - name: ingestair-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ingestair-secret-key.txt + - secrets/sql-password.txt + - secrets/smtp-password.txt + - name: clairchen-forwarder-v2-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/clairchen-forwarder-v2-access-key.txt + - name: ers-forwarder-v2-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ers-forwarder-v2-access-key.txt + - name: clairchen-forwarder-v3-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/clairchen-forwarder-v3-access-key.txt + - name: ers-forwarder-v3-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ers-forwarder-v3-access-key.txt diff --git a/k8s/environments/syseleven/95-cert-issuer.yaml b/k8s/environments/syseleven/95-cert-issuer.yaml new file mode 100644 index 0000000..c839482 --- /dev/null +++ b/k8s/environments/syseleven/95-cert-issuer.yaml @@ -0,0 +1,14 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: jan.weil@web.de + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: nginx diff --git a/k8s/environments/syseleven/add_tls.yaml b/k8s/environments/syseleven/add_tls.yaml new file mode 100644 index 0000000..3c8caf8 --- /dev/null +++ b/k8s/environments/syseleven/add_tls.yaml @@ -0,0 +1,52 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: clair-berlin + annotations: + cert-manager.io/cluster-issuer: letsencrypt-prod +spec: + tls: + - hosts: + - clair.jawebada.de + secretName: ingress-tls-secret + rules: + - host: clair.jawebada.de + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: static-frontend + port: + number: 80 + - path: /admin + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /static + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /accounts + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + - path: /api + pathType: Prefix + backend: + service: + name: managair-server + port: + number: 8888 + diff --git a/k8s/environments/syseleven/config.env b/k8s/environments/syseleven/config.env new file mode 100644 index 0000000..37f3679 --- /dev/null +++ b/k8s/environments/syseleven/config.env @@ -0,0 +1,24 @@ +CLAIR_DOMAIN=clair.jawebada.de + +DEBUG=0 +SENTRY=1 +DEBUG_TOOLBAR=0 + +DJANGO_LOG_LEVEL=WARNING +DJANGO_DB_LOG_LEVEL=WARNING +LOG_LEVEL=WARNING + +MANAGAIR_DB_MIGRATE=true +MANAGAIR_COLLECT_STATIC_FILES=true + +INGESTAIR_DB_MIGRATE=false + +SQL_DATABASE=managairdb_dev +SQL_USER=managair_dev +SQL_PASSWORD=postgres + +EMAIL_HOST=mx2ed5.netcup.net +EMAIL_PORT=587 +EMAIL_HOST_USER=clair-sender@clair-berlin.de +EMAIL_USE_TLS=True +DEFAULT_FROM_EMAIL=kontakt@clair-berlin.de diff --git a/k8s/environments/syseleven/kustomization.yaml b/k8s/environments/syseleven/kustomization.yaml new file mode 100644 index 0000000..9e9246c --- /dev/null +++ b/k8s/environments/syseleven/kustomization.yaml @@ -0,0 +1,55 @@ +resources: + - ../../base + - 95-cert-issuer.yaml + +patchesStrategicMerge: + - add_tls.yaml + +configMapGenerator: + - name: clair-config-map + namespace: clair-berlin + behavior: replace + envs: + - config.env + +secretGenerator: + - name: db-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/sql-password.txt + - name: managair-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/managair-secret-key.txt + - secrets/sql-password.txt + - secrets/sentry-url.txt + - secrets/smtp-password.txt + - name: ingestair-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ingestair-secret-key.txt + - secrets/sql-password.txt + - secrets/smtp-password.txt + - name: clairchen-forwarder-v2-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/clairchen-forwarder-v2-access-key.txt + - name: ers-forwarder-v2-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ers-forwarder-v2-access-key.txt + - name: clairchen-forwarder-v3-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/clairchen-forwarder-v3-access-key.txt + - name: ers-forwarder-v3-secret + namespace: clair-berlin + behavior: replace + files: + - secrets/ers-forwarder-v3-access-key.txt diff --git a/tools/k8s-dump-db.sh b/tools/k8s-dump-db.sh new file mode 100755 index 0000000..f219b30 --- /dev/null +++ b/tools/k8s-dump-db.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env zsh + +DB_POD=$(kubectl --namespace clair-berlin get pod -l app=db -o name) +DB_POD=${DB_POD#pod/} + +kubectl --namespace clair-berlin exec $DB_POD -- pg_dump -U managair_dev managairdb_dev diff --git a/tools/k8s-restore-db-dump.sh b/tools/k8s-restore-db-dump.sh new file mode 100755 index 0000000..c9eab56 --- /dev/null +++ b/tools/k8s-restore-db-dump.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env zsh + +SQL_PATH=$1 + +if [ -z "$SQL_PATH" -o ! -f $SQL_PATH ]; then + echo "file '$SQL_PATH' does not exist" + exit 1 +fi + +SQL_FILE=$(basename $SQL_PATH) + +function scale_db_deployments () { + echo "scaling db deployments to $1" + for deployment in ers-forwarder-v3 ers-forwarder-v2 clairchen-forwarder-v3 clairchen-forwarder-v2 ingestair managair-server; do + kubectl --namespace clair-berlin scale deployment $deployment --replicas $1 + done +} + +scale_db_deployments 0 + +DB_POD=$(kubectl --namespace clair-berlin get pod -l app=db -o name) +DB_POD=${DB_POD#pod/} + +kubectl --namespace clair-berlin cp $SQL_PATH clair-berlin/$DB_POD:/tmp +kubectl --namespace clair-berlin exec $DB_POD -- dropdb -U managair_dev managairdb_dev +kubectl --namespace clair-berlin exec $DB_POD -- createdb -U managair_dev managairdb_dev +kubectl --namespace clair-berlin exec $DB_POD -- bash -c "psql -U managair_dev managairdb_dev < /tmp/$SQL_FILE" + +scale_db_deployments 1