Added s3 endpoint to postgres-restore-s3

nginx-proxy/Dockerfile

@@ -0,0 +1,11 @@
+FROM gcc as builder
+RUN apt update && apt install make libpcre3-dev zlib1g-dev unzip git
+WORKDIR /build
+RUN wget -qO - http://nginx.org/download/nginx-1.17.5.tar.gz | tar zxfv -
+RUN git clone https://github.com/openresty/headers-more-nginx-module
+RUN cd nginx-1.17.5 && ./configure --prefix=/opt/nginx --with-compat --add-dynamic-module=/build/headers-more-nginx-module && make modules
+RUN ls /build/nginx-1.17.5/objs/
+
+FROM jwilder/nginx-proxy:alpine
+COPY --from=builder /build/nginx-1.17.5/objs/ngx_http_headers_more_filter_module.so /usr/lib/nginx/modules/
+RUN sed -i "/user  nginx;/aload_module modules/ngx_http_headers_more_filter_module.so;" /etc/nginx/nginx.conf

nginx-proxy/README.md

@@ -0,0 +1,3 @@
+This is a version of jwilder/nginx-proxy with 'headers more'-module installed.
+
+Example of usage is in `docker-compose.yml` and `/vhost/`

nginx-proxy/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "2.0"
+services:
+  nginx-proxy:
+    image: nginx-proxy:1.17-headers-more
+    ports:
+      - "80:80"
+    volumes:
+      - /var/run/docker.sock:/tmp/docker.sock:ro
+      - ./vhost:/etc/nginx/vhost.d:ro
+
+  whoami:
+    image: jwilder/whoami
+    environment:
+      - VIRTUAL_HOST=whoami.local

nginx-proxy/vhost/whoami.local_location

@@ -0,0 +1,4 @@
+        more_set_headers 'Access-Control-Allow-Origin: http://whoami.local';
+        more_set_headers 'Access-Control-Allow-Methods: GET';
+        more_set_headers 'Access-Control-Allow-Headers: User-Agent,Keep-Alive,Content-Type';
+        more_set_headers 'Access-Control-Expose-Headers: Content-Length,Content-Range';

postgres-backup-s3/Dockerfile

@@ -18,6 +18,10 @@ ENV S3_PATH 'backup'
 ENV S3_ENDPOINT **None**
 ENV S3_S3V4 no
 ENV SCHEDULE **None**
+# Telegram alerting
+ENV BOT_ID **None**
+ENV CHAT_ID **None**
+ENV MSG ''
 
 ADD run.sh run.sh
 ADD backup.sh backup.sh

postgres-backup-s3/backup.sh

@@ -59,10 +59,14 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTG
 
 echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..."
 
-pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | gzip > dump.sql.gz
+pg_dump $POSTGRES_HOST_OPTS -Fc $POSTGRES_DATABASE | gzip > dump.sql.gz
 
 echo "Uploading dump to $S3_BUCKET"
 
-cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2
+if [ "${BOT_ID}" == "**None**" ] || [ "$CHAT_ID" == "**None**" ] || [ -z "$MSG" ]  ; then
+    cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2
+else
+    cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || (curl "https://api.telegram.org/$BOT_ID/sendMessage?chat_id=$CHAT_ID&parse_mode=html&text=$MSG" && exit 2) || exit 2
+fi
 
 echo "SQL backup uploaded successfully"

postgres-restore-s3/Dockerfile

@@ -9,11 +9,13 @@ ENV POSTGRES_HOST **None**
 ENV POSTGRES_PORT 5432
 ENV POSTGRES_USER **None**
 ENV POSTGRES_PASSWORD **None**
+ENV RESTORE_JOBS 4
 ENV S3_ACCESS_KEY_ID **None**
 ENV S3_SECRET_ACCESS_KEY **None**
 ENV S3_BUCKET **None**
 ENV S3_REGION us-west-1
 ENV S3_PATH 'backup'
+ENV S3_ENDPOINT **None**
 ENV DROP_PUBLIC 'no'
 
 ADD restore.sh restore.sh

postgres-restore-s3/restore.sh

@@ -1,6 +1,7 @@
 #! /bin/sh
 
 set -e
+set -x
 set -o pipefail
 
 if [ "${S3_ACCESS_KEY_ID}" = "**None**" ]; then
@@ -43,6 +44,12 @@ if [ "${POSTGRES_PASSWORD}" = "**None**" ]; then
   exit 1
 fi
 
+if [ "${S3_ENDPOINT}" == "**None**" ]; then
+  AWS_ARGS=""
+else
+  AWS_ARGS="--endpoint-url ${S3_ENDPOINT}"
+fi
+
 # env vars needed for aws tools
 export AWS_ACCESS_KEY_ID=$S3_ACCESS_KEY_ID
 export AWS_SECRET_ACCESS_KEY=$S3_SECRET_ACCESS_KEY
@@ -53,21 +60,32 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER"
 
 echo "Finding latest backup"
 
-LATEST_BACKUP=$(aws s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }')
+LATEST_BACKUP=$(aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | sort | tail -n 1 | awk '{ print $4 }')
 
 echo "Fetching ${LATEST_BACKUP} from S3"
 
-aws s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz
+aws $AWS_ARGS s3 cp s3://$S3_BUCKET/$S3_PREFIX/${LATEST_BACKUP} dump.sql.gz
 gzip -d dump.sql.gz
 
 if [ "${DROP_PUBLIC}" == "yes" ]; then
 	echo "Recreating the public schema"
-	psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE -c "drop schema public cascade; create schema public;"
+	psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "drop schema public cascade; create schema public;"
 fi
 
-echo "Restoring ${LATEST_BACKUP}"
+echo "Restoring ${LATEST_BACKUP} to ${POSTGRES_DATABASE} on ${POSTGRES_HOST}:${POSTGRES_PORT} with user ${POSTGRES_USER}"
 
-psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE < dump.sql
+set +e
+psql $POSTGRES_HOST_OPTS -d postgres -c "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$POSTGRES_DATABASE' AND pid <> pg_backend_pid();"
+psql $POSTGRES_HOST_OPTS -d postgres -c "DROP DATABASE IF EXISTS \"$POSTGRES_DATABASE\";"
+psql $POSTGRES_HOST_OPTS -d postgres -c "CREATE DATABASE \"$POSTGRES_DATABASE\";"
+psql $POSTGRES_HOST_OPTS -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"$POSTGRES_DATABASE\" TO $POSTGRES_USER;"
+
+set -e
+psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS citext;"
+psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "ALTER EXTENSION citext SET SCHEMA public;"
+psql $POSTGRES_HOST_OPTS -d "$POSTGRES_DATABASE" -c "CREATE EXTENSION IF NOT EXISTS plpgsql;"
+pg_restore -V
+pg_restore $POSTGRES_HOST_OPTS --no-owner --single-transaction -n public --no-privileges -v -d "$POSTGRES_DATABASE" dump.sql
 
 echo "Restore complete"