Merge branch 'refactor/docker-compose' into dev

Author: eidens

.env.example

@@ -2,9 +2,17 @@ NEO_URI=bolt://neo4j:7687
 NEO_USERNAME=neo4j
 NEO_PASSWORD=sourcedata
 NEO4J_AUTH=neo4j/sourcedata
-NEO4J_dbms_memory_pagecache_size=2G
-NEO4J_dbms_memory_heap_max__size=1G
+NEO4J_MEMORY_LIMIT=2G
+
+NEO4J_dbms_checkpoint_interval_time=120s
+NEO4J_dbms_checkpoint_interval_tx=10000
+NEO4J_dbms_checkpoint_iops_limit=-1
+NEO4J_dbms_jvm_additional=-Dlog4j2.formatMsgNoLookups=True
+NEO4J_dbms_memory_heap_initial__size=500M
+NEO4J_dbms_memory_heap_max__size=500M
+NEO4J_dbms_memory_pagecache_size=1G
 NEO4J_dbms_security_procedures_unrestricted=gds.*
+NEO4J_dbms_tx__log_rotation_retention__policy=keep_none
 
 SD_API_URL=https://api.sourcedata.io/
 SD_API_USERNAME=
@@ -15,15 +23,23 @@ EEB_PUBLIC_API=https://eeb.embo.org/api/
 
 SMTAG_WORKING_DIRECTORY=./resources
 SMTAG_PRODUCTION_DIRECTORY=./rack
-NBITS=8
 
 HYPOTHESIS_USER=
 HYPOTHESIS_API_KEY=
+OPENAI_API_TOKEN=
 
-TRAEFIK_HOSTNAME=
-TRAEFIK_HOSTNAME_OLD=
+BIORXIV_PATH=./data/dgx/biorxiv
+MECADOI_PATH=./data/dgx/mecadoi
 
-IMAGE_TAG=dev
+SMTP_HOST=
+SMTP_STARTTLS_PORT=
+SMTP_USERNAME=
+SMTP_PASSWORD=
+NOTIFICATIONS_RECIPIENT=
+NOTIFICATIONS_SENDER=
 
 FLOW_BATCH_SIZE=100
-DB_DEPLOY_EMAILS=user@example.com
+
+COMPOSE_HTTP_TIMEOUT=600
+IMAGE_TAG=dev
+DB_DEPLOY_EMAILS=user@example.com

.env.example.prod

@@ -0,0 +1,13 @@
+NEO_URI=
+NEO_USERNAME=
+NEO_PASSWORD=
+NEO4J_AUTH=
+NEO4J_dbms_memory_heap_initial__size=
+NEO4J_dbms_memory_heap_max__size=
+NEO4J_dbms_memory_pagecache_size=
+NEO4J_dbms_security_procedures_unrestricted=gds.*
+NEO4J_dbms_jvm_additional=-Dlog4j2.formatMsgNoLookups=True
+NEO4J_dbms_allow__upgrade=true
+
+EEB_HOSTNAME=eeb.embo.org
+EEB_HOSTNAME_OLD=eeb.sourcedata.io

.github/workflows/ci.yml

@@ -68,7 +68,7 @@ jobs:
     if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/dev'
     strategy:
       matrix:
-        image: ['flask_deploy', 'flask', 'traefik', 'vuejs']
+        image: ['flask', 'traefik', 'vuejs']
     needs: test
     runs-on: ubuntu-latest
 
@@ -88,8 +88,7 @@ jobs:
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Build the Docker images
-      run: docker compose -f docker-compose.ci.yml build ${{ matrix.image }}
+      run: docker compose -f docker-compose.yml -f docker-compose.prod.yml build ${{ matrix.image }}
 
     - name: Push the Docker images
-      # retry up to 3 times
-      run: docker compose -f docker-compose.ci.yml push ${{ matrix.image }} || docker compose -f docker-compose.ci.yml push ${{ matrix.image }} || docker compose -f docker-compose.ci.yml push ${{ matrix.image }}
+      run: docker compose -f docker-compose.yml -f docker-compose.prod.yml push ${{ matrix.image }}

README.md

@@ -22,9 +22,26 @@ docker-compose down --volumes # to clean the content of the volumes
 
 This can solve some issues, for example if you run `build` with a wrong config file.
 
+### Compose files
+
+This repository contains multiple docker-compose files that are [merged together](https://docs.docker.com/compose/how-tos/multiple-compose-files/merge/) to define the services and their configurations for different environments (production, testing). The files are:
+
+- `docker-compose.yml`: the main compose file that defines the services and their dependencies. This file is used as the base configuration for all environments.
+- `docker-compose.override.yml`: the override compose file that extends the main compose file with additional configurations for development. It is automatically used by Docker Compose if present and no other compose files are specified. This file is used both for local development and for running the database deployment.
+- `docker-compose.prod.yml`: the production compose file that extends the main compose file with production-specific configurations.
+- `docker-compose.tests.yml`: the tests compose file that extends the main compose file with configurations for running tests. Used in `scripts/run-tests.sh` and CI.
+
+If you run `docker-compose up` (or any other sub-command) without specifying a compose file, `docker-compose` will use the `docker-compose.yml` and `docker-compose.override.yml` files by default. If you want to run the production setup, use this command:
+
+```bash
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml up
+```
+
 ### First time setup:
 
 ```bash
+# copy the example .env file and edit with your desired config
+# .env.example.prod is for the production webserver
 cp .env.example .env
 docker-compose build
 docker-compose up
@@ -145,7 +162,7 @@ docker run --rm \
 
 ```bash
 # Make sure you dont have your neo4j running:
-docker-compose -f production.yml down
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml down
 
 sudo mkdir dumps
 sudo chown 7474:7474 dumps
@@ -188,7 +205,7 @@ wget https://oc.embl.de/index.php/s/<token>/download
 
 
 # build docker
-docker-compose -f production.yml build
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
 
 # force load the database with the dump
 # THIS WILL OVERWRITE THE EXISTING DB
@@ -201,8 +218,8 @@ docker run --rm \
  bin/neo4j-admin load --from=/app/download --database=neo4j --force  # WILL OVERWRITE!
 
 # start the services
-docker-compose -f production.yml up -d --remove-orphans
-docker-compose -f production.yml run --rm flask python -m neoflask.cache_warm_up  # warm up cache
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --remove-orphans
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml run --rm flask python -m neoflask.cache_warm_up  # warm up cache
 ```
 
 
@@ -211,6 +228,6 @@ Something like this will (generally) be enough, but really depends on your chang
 
 ```bash
 git pull
-docker-compose -f production.yml build
-docker-compose -f production.yml up -d --remove-orphans
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml build
+docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d --remove-orphans
 ```