diff --git a/.dclintrc b/.dclintrc new file mode 100644 index 0000000000..c2a5f09e52 --- /dev/null +++ b/.dclintrc @@ -0,0 +1,8 @@ +{ + "rules": { + "require-project-name-field": 0, + "no-build-and-image": 0 + }, + "quiet": false, + "debug": true +} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 331659c7be..a548c6ea0c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -14,9 +14,11 @@ name: "CodeQL" on: push: branches: [ master, production ] + paths: [ TWLight ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] + paths: [ TWLight ] schedule: - cron: '31 13 * * 3' @@ -39,7 +41,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -50,7 +52,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@v4 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -64,4 +66,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dclint.yml b/.github/workflows/dclint.yml new file mode 100644 index 0000000000..7804af7cf0 --- /dev/null +++ b/.github/workflows/dclint.yml @@ -0,0 +1,15 @@ +name: DCLint + +on: + push: + branches: [ master, production ] + pull_request: + branches: [ master ] + types: [ synchronize ] + +jobs: + dclint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker-compose-linter/dclint-github-action/docker-action@v1.6.0 diff --git a/bin/debian_swarm_deploy.sh b/bin/debian_swarm_deploy.sh index 14ef152e0c..e1d978d352 100755 --- a/bin/debian_swarm_deploy.sh +++ b/bin/debian_swarm_deploy.sh @@ -41,7 +41,7 @@ git clone https://github.com/WikipediaLibrary/TWLight.git /srv/TWLight cd /srv/TWLight || exit # Get on correct branch echo "Enter git branch:" -read TWLIGHT_GIT_BRANCH +read -r TWLIGHT_GIT_BRANCH git checkout "${TWLIGHT_GIT_BRANCH}" && git pull # Get input from human diff --git a/bin/twlight_docker_deploy.sh b/bin/twlight_docker_deploy.sh index 726284c24d..bf2307ce35 100755 --- a/bin/twlight_docker_deploy.sh +++ b/bin/twlight_docker_deploy.sh @@ -12,17 +12,23 @@ env=${1} tag=${2} # Move into the repository. -cd /srv/TWLight +cd /srv/TWLight || (echo "Error"; exit 1) # Check for newer image -pull=$(docker pull quay.io/wikipedialibrary/twlight:${tag}) +pull=$(docker pull "quay.io/wikipedialibrary/twlight:${tag}") # Pull swarm config updates and update the stack if there is a new image. -if echo ${pull} | grep "Status: Downloaded newer image for quay.io/wikipedialibrary/twlight:${tag}" >/dev/null +if echo "${pull}" | grep "Status: Downloaded newer image for quay.io/wikipedialibrary/twlight:${tag}" >/dev/null then + # Accept divergent commit history from staging remote + if [ "$env" == "staging" ] + then + git fetch + git reset --hard origin + fi git pull - docker stack deploy -c docker-compose.yml -c docker-compose.${env}.yml ${env} + docker stack deploy --detach=false -c docker-compose.yml -c "docker-compose.${env}.yml" "${env}" # Report if the local image is already up to date. -elif echo ${pull} | grep "Status: Image is up to date for quay.io/wikipedialibrary/twlight:${tag}" >/dev/null +elif echo "${pull}" | grep "Status: Image is up to date for quay.io/wikipedialibrary/twlight:${tag}" >/dev/null then echo "Up to date" # Fail in any other circumstance. diff --git a/docker-compose.cicd.yml b/docker-compose.cicd.yml index 5d7e610148..f700476aa2 100644 --- a/docker-compose.cicd.yml +++ b/docker-compose.cicd.yml @@ -1,7 +1,4 @@ --- - -version: '3.4' - services: twlight: image: quay.io/wikipedialibrary/twlight:local diff --git a/docker-compose.debug_toolbar.yml b/docker-compose.debug_toolbar.yml index 0a69b18a40..fa7d81aba4 100644 --- a/docker-compose.debug_toolbar.yml +++ b/docker-compose.debug_toolbar.yml @@ -1,7 +1,4 @@ --- - -version: '3.4' - services: twlight: build: diff --git a/docker-compose.override.yml b/docker-compose.override.yml index f406ba81d8..12567e99c1 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,8 +1,24 @@ --- - -version: '3.4' - -# Local environment should mount plaintext files as secrets +services: + twlight: + image: quay.io/wikipedialibrary/twlight:local + # Local environment should mount things from the code directory + volumes: + - type: bind + source: . + target: /app + env_file: + - path: ./conf/local.twlight.env + required: true + - path: .env + required: false + extra_hosts: + - "host.docker.internal:host-gateway" + web: + volumes: + - type: bind + source: ./conf/local.nginx.conf + target: /etc/nginx/conf.d/default.conf secrets: DJANGO_DB_NAME: file: ./secrets/DJANGO_DB_NAME @@ -26,24 +42,3 @@ secrets: file: ./secrets/MW_API_EMAIL_USER MW_API_EMAIL_PASSWORD: file: ./secrets/MW_API_EMAIL_PASSWORD - -services: - twlight: - extra_hosts: - - "host.docker.internal:host-gateway" - image: quay.io/wikipedialibrary/twlight:local - env_file: - - path: ./conf/local.twlight.env - required: true - - path: .env - required: false - # Local environment should mount things from the code directory - volumes: - - type: bind - source: . - target: /app - web: - volumes: - - type: bind - source: ./conf/local.nginx.conf - target: /etc/nginx/conf.d/default.conf diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 7a0971061c..faa5d65989 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -1,39 +1,9 @@ --- - -version: '3.4' - -secrets: - DJANGO_DB_NAME: - external: true - DJANGO_DB_USER: - external: true - DJANGO_DB_PASSWORD: - external: true - DKIM_PRIVATE_KEY: - external: true - MYSQL_ROOT_PASSWORD: - external: true - SECRET_KEY: - external: true - TWLIGHT_OAUTH_CONSUMER_KEY: - external: true - TWLIGHT_OAUTH_CONSUMER_SECRET: - external: true - TWLIGHT_EZPROXY_SECRET: - external: true - MW_API_EMAIL_USER: - external: true - MW_API_EMAIL_PASSWORD: - external: true - services: migrate: image: quay.io/wikipedialibrary/twlight:branch_production - command: > - bash -c "/app/bin/twlight_backup.sh && /app/bin/virtualenv_migrate.sh" - restart: on-failure - env_file: - - ./conf/production.twlight.env + depends_on: + - 'db' volumes: - type: bind source: /data/project/prod @@ -41,8 +11,6 @@ services: - type: bind source: ./media target: /app/media - depends_on: - - 'db' secrets: - DJANGO_DB_NAME - DJANGO_DB_USER @@ -52,16 +20,21 @@ services: - TWLIGHT_OAUTH_CONSUMER_KEY - TWLIGHT_OAUTH_CONSUMER_SECRET - TWLIGHT_EZPROXY_SECRET - twlight: - image: quay.io/wikipedialibrary/twlight:branch_production env_file: - ./conf/production.twlight.env + command: > + bash -c "/app/bin/twlight_backup.sh && /app/bin/virtualenv_migrate.sh" + restart: on-failure + twlight: + image: quay.io/wikipedialibrary/twlight:branch_production + depends_on: + - 'migrate' volumes: - type: bind source: /data/project/prod target: /app/backup - depends_on: - - 'migrate' + env_file: + - ./conf/production.twlight.env web: volumes: - type: volume @@ -69,3 +42,26 @@ services: - type: bind source: ./conf/production.nginx.conf target: /etc/nginx/conf.d/default.conf +secrets: + DJANGO_DB_NAME: + external: true + DJANGO_DB_USER: + external: true + DJANGO_DB_PASSWORD: + external: true + DKIM_PRIVATE_KEY: + external: true + MYSQL_ROOT_PASSWORD: + external: true + SECRET_KEY: + external: true + TWLIGHT_OAUTH_CONSUMER_KEY: + external: true + TWLIGHT_OAUTH_CONSUMER_SECRET: + external: true + TWLIGHT_EZPROXY_SECRET: + external: true + MW_API_EMAIL_USER: + external: true + MW_API_EMAIL_PASSWORD: + external: true diff --git a/docker-compose.staging.yml b/docker-compose.staging.yml index 0d7d0238b6..455841d8c1 100644 --- a/docker-compose.staging.yml +++ b/docker-compose.staging.yml @@ -1,39 +1,9 @@ --- - -version: '3.4' - -secrets: - DJANGO_DB_NAME: - external: true - DJANGO_DB_USER: - external: true - DJANGO_DB_PASSWORD: - external: true - DKIM_PRIVATE_KEY: - external: true - MYSQL_ROOT_PASSWORD: - external: true - SECRET_KEY: - external: true - TWLIGHT_OAUTH_CONSUMER_KEY: - external: true - TWLIGHT_OAUTH_CONSUMER_SECRET: - external: true - TWLIGHT_EZPROXY_SECRET: - external: true - MW_API_EMAIL_USER: - external: true - MW_API_EMAIL_PASSWORD: - external: true - services: migrate: image: quay.io/wikipedialibrary/twlight:branch_staging - command: > - bash -c "/app/bin/twlight_backup.sh && /app/bin/virtualenv_migrate.sh" - restart: on-failure - env_file: - - ./conf/staging.twlight.env + depends_on: + - 'db' volumes: - type: bind source: /data/project/staging @@ -41,8 +11,6 @@ services: - type: bind source: ./media target: /app/media - depends_on: - - 'db' secrets: - DJANGO_DB_NAME - DJANGO_DB_USER @@ -52,16 +20,21 @@ services: - TWLIGHT_OAUTH_CONSUMER_KEY - TWLIGHT_OAUTH_CONSUMER_SECRET - TWLIGHT_EZPROXY_SECRET - twlight: - image: quay.io/wikipedialibrary/twlight:branch_staging env_file: - ./conf/staging.twlight.env + command: > + bash -c "/app/bin/twlight_backup.sh && /app/bin/virtualenv_migrate.sh" + restart: on-failure + twlight: + image: quay.io/wikipedialibrary/twlight:branch_staging + depends_on: + - 'migrate' volumes: - type: bind source: /data/project/staging target: /app/backup - depends_on: - - 'migrate' + env_file: + - ./conf/staging.twlight.env web: volumes: - type: volume @@ -69,3 +42,26 @@ services: - type: bind source: ./conf/staging.nginx.conf target: /etc/nginx/conf.d/default.conf +secrets: + DJANGO_DB_NAME: + external: true + DJANGO_DB_USER: + external: true + DJANGO_DB_PASSWORD: + external: true + DKIM_PRIVATE_KEY: + external: true + MYSQL_ROOT_PASSWORD: + external: true + SECRET_KEY: + external: true + TWLIGHT_OAUTH_CONSUMER_KEY: + external: true + TWLIGHT_OAUTH_CONSUMER_SECRET: + external: true + TWLIGHT_EZPROXY_SECRET: + external: true + MW_API_EMAIL_USER: + external: true + MW_API_EMAIL_PASSWORD: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 7a8f3baac8..59c0fd6f49 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,14 +1,6 @@ --- - -version: '3.4' - -volumes: - mysql: - services: db: - ports: - - "3306:3306" image: quay.io/wikipedialibrary/mariadb:10-updated volumes: - mysql:/var/lib/mysql @@ -18,6 +10,11 @@ services: - type: bind source: ./bin/twlight_mysql_init.sh target: /docker-entrypoint-initdb.d/twlight_mysql_init.sh + secrets: + - DJANGO_DB_NAME + - DJANGO_DB_USER + - DJANGO_DB_PASSWORD + - MYSQL_ROOT_PASSWORD environment: # https://mariadb.com/kb/en/mariadb-docker-environment-variables/ - MYSQL_ROOT_PASSWORD_FILE=/run/secrets/MYSQL_ROOT_PASSWORD @@ -25,23 +22,17 @@ services: - MARIADB_AUTO_UPGRADE=TRUE # Any non-empty value will do - MARIADB_DISABLE_UPGRADE_BACKUP=TRUE - secrets: - - DJANGO_DB_NAME - - DJANGO_DB_USER - - DJANGO_DB_PASSWORD - - MYSQL_ROOT_PASSWORD twlight: build: context: . args: - REQUIREMENTS_FILE=wmf.txt - command: ["/venv/bin/gunicorn", "TWLight.wsgi"] + depends_on: + - 'db' volumes: - type: bind source: ./media target: /app/media - depends_on: - - 'db' secrets: - DJANGO_DB_NAME - DJANGO_DB_USER @@ -53,10 +44,11 @@ services: - TWLIGHT_EZPROXY_SECRET - MW_API_EMAIL_USER - MW_API_EMAIL_PASSWORD + command: [ "/venv/bin/gunicorn", "TWLight.wsgi" ] web: - ports: - - "80:80" image: quay.io/wikipedialibrary/nginx:latest-updated + depends_on: + - 'twlight' volumes: - type: bind source: ./conf/robots.txt @@ -70,5 +62,9 @@ services: - type: bind source: ./TWLight/templates/500 target: /app/500 - depends_on: - - 'twlight' + ports: + - '0.0.0.0:80:80' + +volumes: + mysql: +