Merge pull request #10 from dlcs/feature/updates

Tweak dockerfile + add entrypoints

Author: donaldgray

.env.dist

@@ -11,7 +11,7 @@ WEB_SERVER_HOSTNAME=localhost:8000
 ORIGIN_CHUNK_SIZE=8192
 
 # Database
-DATABASE_URL=postgresql://dlcs:password@127.0.0.1:5432/compositedb
+DATABASE_URL=postgresql://dlcs:password@postgres:5432/compositedb
 
 # CACHE
 CACHE_URL=dbcache://app_cache
@@ -33,3 +33,12 @@ ENGINE_WORKER_COUNT=2
 ENGINE_WORKER_TIMEOUT=3600
 ENGINE_WORKER_RETRY=4500
 ENGINE_WORKER_MAX_ATTEMPTS=0
+
+# Run migrations
+MIGRATE=True
+
+# DJANGO ADMIN
+INIT_SUPERUSER=True
+DJANGO_SUPERUSER_EMAIL=x.y@z
+DJANGO_SUPERUSER_PASSWORD=composite-handler-password
+DJANGO_SUPERUSER_USERNAME=Admin

.gitignore

@@ -152,8 +152,35 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# PyCharm
+# User-specific stuff
+.idea/**/workspace.xml
+src/.idea/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# File-based project format
+*.iws
+
+# Editor-based Rest Client
 .idea/
 
 # Temp working files
-src/scratch/
+src/scratch/
+scratch/

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 LABEL maintainer="Donald Gray <donald.gray@digirati.com>"
 LABEL org.opencontainers.image.source=https://github.com/dlcs/composite-handler
 
@@ -13,9 +13,14 @@ RUN apt-get update && apt-get --yes install apt-utils && apt-get --yes upgrade \
     && useradd --create-home --home-dir /srv/dlcs --shell /bin/bash --uid 1000 dlcs \
     && python -m pip install --upgrade pip
 
+COPY --chown=dlcs:dlcs ./src/requirements.txt /srv/dlcs
+RUN pip install --no-warn-script-location --requirement /srv/dlcs/requirements.txt
+
 COPY --chown=dlcs:dlcs ./src /srv/dlcs
+COPY --chown=dlcs:dlcs ./entrypoints /srv/dlcs
+RUN chmod +x /srv/dlcs/entrypoint.sh
+RUN chmod +x /srv/dlcs/entrypoint-api.sh
+RUN chmod +x /srv/dlcs/entrypoint-worker.sh
 
 USER dlcs
 WORKDIR /srv/dlcs
-
-RUN pip install --no-warn-script-location --requirement requirements.txt

README.md

@@ -27,7 +27,7 @@ The project ships with a [`docker-compose.yml`](docker-compose.yml) that can be
 docker-compose up
 ```
 
-Note that for the Composite Handler to be able to interact with the target S3 bucket, the Docker Compose assumes that the `AWS_PROFILE` environment variable has been set and a valid AWS session is available.
+> Note that for the Composite Handler to be able to interact with the target S3 bucket, the Docker Compose assumes that the `AWS_PROFILE` environment variable has been set and a valid AWS session is available.
 
 This will create a PostgreSQL instance, bootstrap it with the required tables, deploy a single instance of the API, and three instances of the engine. Requests can then be targetted at `localhost:8000`.
 
@@ -47,18 +47,26 @@ python manage.py createcachetable
 
 Once the API is running, an administrator interface can be accessed via the browser at `http://localhost:8000/admin`. To create an administrator login, run the following command:
 
-```
+```bash
 python manage.py createsuperuser
 ```
 
 The administrator user can be used to browse the database and manage the queue (including deleting tasks and resubmitting failed tasks into the queue).
 
+### Entrypoints
+
+There are 3 possible entrypoints to make the above easier:
+
+* `entrypoint.sh` - this will wait for Postgres to be available and run `manage.py migrate` and `manage.py createcachetable` if `MIGRATE=True`. It will run `manage.py createsuperuser` is `INIT_SUPERUSER=True` (also needs `DJANGO_SUPERUSER_*` envvars)
+* `entrypoint-api.sh` - this runs above then `python manage.py runserver 0.0.0.0:8000`
+* `entrypoint-worker.sh` - this runs above then `python manage.py qcluster`
+
 ## Configuration
 
 The following list of environment variables are supported:
 
 | Environment Variable          | Default Value                  | Component(s) | Description                                                                                                                                                                                                                                                                  |
-|-------------------------------|--------------------------------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ----------------------------- | ------------------------------ | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | `DJANGO_DEBUG`                | `True`                         | API, Engine  | Whether Django should run in debug. Useful for development purposes but should be set to `False` in production.                                                                                                                                                              |
 | `DJANGO_SECRET_KEY`           | None                           | API, Engine  | The secret key used by Django when generating sensitive tokens. This should a randomly generated 50 character string.                                                                                                                                                        |
 | `SCRATCH_DIRECTORY`           | `/tmp/scratch`                 | Engine       | A locally accessible filesystem path where work-in-progress files are written during rasterization.                                                                                                                                                                          |
@@ -78,6 +86,8 @@ The following list of environment variables are supported:
 | `ENGINE_WORKER_TIMEOUT`       | `3600`                         | Engine       | The number of seconds that a task (i.e. the processing of a single PDF) can run for before being terminated and treated as a failure. This value is useful to purging "stuck" tasks which haven't technically failed but are occupying a worker.                             |
 | `ENGINE_WORKER_RETRY`         | `4500`                         | Engine       | The number of seconds since a task was presented for processing before a worker will re-run, regardless of whether it is still running or failed. As such, this value must be higher than `ENGINE_WORKER_TIMEOUT`.                                                           |
 | `ENGINE_WORKER_MAX_ATTEMPTS`  | `0`                            | Engine       | The number of processing attempts a single task will undergo before it is abandoned. Setting this value to `0` will cause a task to be retried forever.                                                                                                                      |
+| `MIGRATE`                     | None                           | API, Engine  | If "True" will run migrations + createcachetable on startup if entrypoint used.                                                                                                                                                                                              |
+| `INIT_SUPERUSER`              | None                           | API, Engine  | If "True" will attempt to create superuser. Needs standard Django envvars to be set (e.g. `DJANGO_SUPERUSER_USERNAME`, `DJANGO_SUPERUSER_EMAIL`, `DJANGO_SUPERUSER_PASSWORD`) if entrypoint used.                                                                            |
 
 Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials.
 

docker-compose.local.yml

@@ -0,0 +1,16 @@
+version: "3.9"
+
+services:
+  postgres:
+    image: postgres:13.3
+    environment:
+      - POSTGRES_USER=dlcs
+      - POSTGRES_PASSWORD=password
+      - POSTGRES_DB=compositedb
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+
+volumes:
+  postgres_data:

docker-compose.yml

@@ -6,23 +6,18 @@ services:
     command: chown -R 1000:1000 /data/scratch
     volumes:
       - ./scratch:/data/scratch
-  migrate:
-    build: .
-    command: bash -c "python manage.py migrate && python manage.py createcachetable"
-    env_file: .env
-    depends_on:
-      - postgres
   api:
     build: .
-    command: python manage.py runserver 0.0.0.0:8000
+    command: /srv/dlcs/entrypoint-api.sh
     env_file: .env
+    environment:
+      - MIGRATE=True
+      - INIT_SUPERUSER=True
     ports:
       - "8000:8000"
-    depends_on:
-      - migrate
   engine:
     build: .
-    command: python manage.py qcluster
+    command: /srv/dlcs/entrypoint-worker.sh
     deploy:
       replicas: 3
     env_file: .env
@@ -32,7 +27,6 @@ services:
       - ./scratch:/data/scratch
       - $HOME/.aws:/srv/dlcs/.aws:ro
     depends_on:
-      - migrate
       - init
   postgres:
     image: postgres:13.3

entrypoints/entrypoint-api.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+bash entrypoint.sh
+python manage.py runserver 0.0.0.0:8000

entrypoints/entrypoint-worker.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+bash entrypoint.sh
+python manage.py qcluster

entrypoints/entrypoint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+POSTGRES_USER=$(echo $DATABASE_URL | grep -oP "postgresql://\K(.+?):" | cut -d: -f1)
+POSTGRES_PASSWORD=$(echo $DATABASE_URL | grep -oP "postgresql://.*:\K(.+?)@" | cut -d@ -f1)
+POSTGRES_HOST=$(echo $DATABASE_URL | grep -oP "postgresql://.*@\K(.+?):" | cut -d: -f1)
+POSTGRES_PORT=$(echo $DATABASE_URL | grep -oP "postgresql://.*@.*:\K(\d+)/" | cut -d/ -f1)
+POSTGRES_DB=$(echo $DATABASE_URL | grep -oP "postgresql://.*@.*:.*/\K(.+?)$")
+
+# Loop until postgres is ready
+postgres_ready() {
+python3 << END
+import sys
+
+import psycopg2
+
+try:
+    psycopg2.connect(
+        dbname="${POSTGRES_DB}",
+        user="${POSTGRES_USER}",
+        password="${POSTGRES_PASSWORD}",
+        host="${POSTGRES_HOST}",
+        port="${POSTGRES_PORT}",
+    )
+except psycopg2.OperationalError:
+    sys.exit(-1)
+sys.exit(0)
+
+END
+}
+until postgres_ready; do
+  >&2 echo 'entrypoint: Waiting for PostgreSQL to become available...'
+  sleep 1
+done
+>&2 echo 'entrypoint: PostgreSQL is available'
+
+if [[ ($MIGRATE) && ("$MIGRATE" = "True") ]]; then
+  python manage.py migrate --no-input
+  echo "entrypoint: Migrations finished"
+  python manage.py createcachetable
+  echo "entrypoint: Create cache table finished"
+fi
+
+if [[ ($INIT_SUPERUSER) && ("$INIT_SUPERUSER" = "True") ]]; then
+  echo "entrypoint: Creating superuser"
+  if python3 manage.py createsuperuser --noinput; then
+    echo "entrypoint: Created superuser"
+  else
+    echo "entrypoint: Superuser already exists, unset INIT_SUPERUSER"
+  fi
+fi

src/app/engine/tasks.py

@@ -1,3 +1,4 @@
+import logging
 import shutil
 
 from app.common.dlcs import DLCS
@@ -9,14 +10,18 @@
 from app.engine.serializers import DLCSBatchSerializer
 from django_q.tasks import async_task
 
+logger = logging.Logger(__name__)
 http_origin = HttpOrigin()
 pdf_rasterizer = PdfRasterizer()
 s3_client = S3Client()
 dlcs = DLCS()
 
 
 def cleanup_scratch(folder_path):
-    shutil.rmtree(folder_path)
+    try:
+        shutil.rmtree(folder_path)
+    except FileNotFoundError:
+        logger.info(f"cleanup failed for {folder_path}, not found")
 
 
 def process_member(args):