cherry-pick: add database migration

Author: davidcheung

README.md

@@ -31,6 +31,8 @@ This repository is language/business-logic agnostic; mainly showcasing some univ
 |       |   |-- deployment.yml
 |       |   |-- kustomization.yml
 |       |   |-- service.yml
+│       ├── migration/
+│       │   └── job.yml
 |       |-- overlays/
 |       |   |-- production/
 |       |   |   |-- deployment.yml

templates/.circleci/config.yml

@@ -249,6 +249,25 @@ jobs:
             DEPLOYMENT=<< parameters.repo >>
             NAMESPACE=<< parameters.namespace >>
             kubectl create namespace $NAMESPACE || echo "Namespace already exists"
+
+            MIGRATION_NAME=<< parameters.repo >>-migration
+            SQL_DIR="${PWD}/database/migration"
+            pushd kubernetes/migration
+            kubectl -n $NAMESPACE delete configmap $MIGRATION_NAME || echo "no migration configmap existing for deletion" 
+            if [ -f ${SQL_DIR}/*.sql ] ; then
+                kubectl -n $NAMESPACE create configmap      $MIGRATION_NAME --from-file ${SQL_DIR}/*.sql
+            else
+                kubectl -n $NAMESPACE create configmap      $MIGRATION_NAME
+            fi
+            kubectl -n $NAMESPACE delete job $MIGRATION_NAME || echo "no migration job existing for deletion" 
+            kubectl -n $NAMESPACE create -f job.yml
+            if ! kubectl -n $NAMESPACE wait --for=condition=complete --timeout=180s job/$MIGRATION_NAME ; then
+                echo "$MIGRATION_NAME run failed:"
+                kubectl -n $NAMESPACE describe job $MIGRATION_NAME
+                exit 1
+            fi
+            popd
+
             cd kubernetes/overlays/<< parameters.config-environment >>
             IMAGE=<< parameters.account-id >>.dkr.ecr.<< parameters.region >>.amazonaws.com/<< parameters.repo >>
             kustomize edit set image fake-image=${IMAGE}:${VERSION_TAG}

templates/README.md

@@ -47,6 +47,37 @@ You likely want to run processes dependent on your backend codebase; so the imag
 As per the image attribute noted above, you will likely be running custom arguments in the context of that image.
 You should specify those arguments [as per the documentation](https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/).
 
+## Database Migration
+Database migrations are handled with [Flyway](https://flywaydb.org/). Migrations run in a docker container started in the Kubernetes cluster by CircleCI or the local dev environment startup process.
+
+The migration job is defined in `kubernetes/migration/job.yml` and your SQL scripts should be in `database/migration/`.
+Migrations will be automatically run against your dev environment when running `./start-dev-env.sh`. After merging the migration it will be run against other environments automatically as part of the pipeline.
+
+The SQL scripts need to follow Flyway naming convention [here](https://flywaydb.org/documentation/concepts/migrations.html#sql-based-migrations), which allow you to create different types of migrations:
+* Versioned - These have a numerically incrementing version id and will be kept track of by Flyway. Only versions that have not yet been applied will be run during the migration process.
+* Undo - These have a matching version to a versioned migration and can be used to undo the effects of a migration if you need to roll back.
+* Repeatable - These will be run whenenver their content changes. This can be useful for seeding data or updating views or functions.
+
+Here are some example migrations:
+
+`V1__create_tables.sql`
+```sql
+CREATE TABLE address (
+    id INT(6) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    person_id INT(6),
+    street_number INT(10),
+    street_name VARCHAR(50),
+    reg_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
+);
+```
+
+`V2__add_columns.sql`
+```sql
+ALTER TABLE address
+ ADD COLUMN city VARCHAR(30) AFTER street_name,
+ ADD COLUMN province VARCHAR(30) AFTER city
+```
+
 <!-- Links -->
 [base-cronjob]: ./kubernetes/base/cronjob.yml
 [base-deployment]: ./kubernetes/base/deployment.yml

templates/database/migration/.migrations-go-here

@@ -0,0 +1,36 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: <% .Name %>-migration
+spec:
+  template:
+    spec:
+      containers:
+        - name: flyway
+          image: flyway/flyway:7.4.0
+          args:
+            - info
+            - repair
+            - migrate
+            - info
+          env:
+            - name: FLYWAY_URL
+              value: jdbc:<% index .Params `database` %>://database.<% .Name %>/<% index .Params `databaseName` %>
+            - name: FLYWAY_USER
+              valueFrom:
+                secretKeyRef:
+                  name: <% .Name %>
+                  key: DATABASE_USERNAME
+            - name: FLYWAY_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: <% .Name %>
+                  key: DATABASE_PASSWORD
+          volumeMounts:
+            - mountPath: /flyway/sql
+              name: sqlfiles
+      volumes:
+        - name: sqlfiles
+          configMap:
+            name: <% .Name %>-migration
+      restartPolicy: Never