docker ops cleanup

.cicd/deploy.sh

@@ -23,35 +23,35 @@ twlight_i18n_files_changed=$((twlight_i18n_files_added+twlight_i18n_files_update
 
 # Add remote and checkout branch
 git_config() {
-    echo ${FUNCNAME[0]}
+    echo "${FUNCNAME[0]}"
     # Configure user.
     git config --global user.email "41753804+WikipediaLibraryBot@users.noreply.github.com"
     git config --global user.name "WikipediaLibraryBot"
-    # Add our authenticated remote using encrypted travis environment variables.
-    git remote add deploy https://WikipediaLibraryBot:${wikibot_token}@github.com/${gh_repo}.git > /dev/null 2>&1
+    # Add our authenticated remote using secret variables.
+    git remote add deploy "https://WikipediaLibraryBot:${wikibot_token}@github.com/${gh_repo}.git" > /dev/null 2>&1
 }
 # Commit migrations
 git_migrate() {
-    echo ${FUNCNAME[0]}
+    echo "${FUNCNAME[0]}"
     msg="${commit} migrations"
     git add 'TWLight/*/migrations/*.py'
     git commit --message "commit ${msg}" ||:
-    git fetch deploy ${branch} --quiet
-    git merge --strategy recursive -X theirs deploy/${branch} --message "merge ${msg}" --quiet
+    git fetch deploy "${branch}" --quiet
+    git merge --strategy recursive -X theirs "deploy/${branch}" --message "merge ${msg}" --quiet
 }
 # Commit translations
 git_i18n() {
-    echo ${FUNCNAME[0]}
+    echo "${FUNCNAME[0]}"
     msg="${commit} translations"
     git add 'locale/*/LC_MESSAGES/*.po'
     git add 'locale/*/LC_MESSAGES/*.mo'
     git commit --message "commit ${msg}" ||:
-    git fetch deploy ${branch} --quiet
-    git merge --strategy recursive -X ours deploy/${branch} --message "merge ${msg}" --quiet
+    git fetch deploy "${branch}" --quiet
+    git merge --strategy recursive -X ours "deploy/${branch}" --message "merge ${msg}" --quiet
 }
 # Commit prod changes
 git_prod() {
-    echo ${FUNCNAME[0]}
+    echo "${FUNCNAME[0]}"
     deploy_branch="${commit}-deployment"
     msg="${commit} deployment"
     git checkout -b "${deploy_branch}"
@@ -62,7 +62,7 @@ git_prod() {
 }
 # Push built images to quay.io
 docker_push() {
-    echo ${FUNCNAME[0]}
+    echo "${FUNCNAME[0]}"
     echo "$quaybot_password" | docker login quay.io -u "$quaybot_username" --password-stdin
     branch_tag="branch_${branch}"
     if [ "${branch}" = "master" ]
@@ -73,8 +73,8 @@ docker_push() {
     declare -a repositories=("twlight")
     for repo in "${repositories[@]}"
     do
-      docker push quay.io/wikipedialibrary/${repo}:commit_${commit}
-      docker push quay.io/wikipedialibrary/${repo}:${branch_tag}
+      docker push "quay.io/wikipedialibrary/${repo}:commit_${commit}"
+      docker push "quay.io/wikipedialibrary/${repo}:${branch_tag}"
     done
     docker logout quay.io
 }
@@ -85,7 +85,7 @@ then
     if [ "${twlight_i18n_files_changed}" -gt 0 ] || [ "${twlight_missing_migrations}" -gt 0 ]
     then
         # Checkout branch
-        git checkout ${branch}
+        git checkout "${branch}"
         # commit and merge any missing migrations
         if [ "${twlight_missing_migrations}" -gt 0 ]
         then
@@ -96,7 +96,7 @@ then
         then
             git_i18n
         fi
-        git push --set-upstream deploy ${branch} --quiet
+        git push --set-upstream deploy "${branch}" --quiet
         echo "pushed to ${branch}"
     elif [ "${twlight_missing_migrations}" -eq 0 ] && [ "${twlight_i18n_files_changed}" -eq 0 ]
     then

.coveragerc

@@ -0,0 +1,9 @@
+[coverage:run]
+plugins = django_coverage_plugin
+source = TWLight
+branch = True
+
+[coverage:report]
+fail_under = 10
+show_missing = True
+skip_covered = True

.github/workflows/cicd.yml

@@ -24,15 +24,15 @@ jobs:
       - name: migrate
         run: >
           docker compose exec --no-TTY twlight
-          /app/bin/virtualenv_migrate.sh
+          bin/virtualenv_migrate.sh
       - name: translate
         run: >
           docker compose exec --no-TTY twlight
-          /app/bin/virtualenv_translate.sh last-commit
+          bin/virtualenv_translate.sh last-commit
       - name: test
         run: >
           docker compose exec --no-TTY twlight
-          /app/bin/virtualenv_test.sh
+          bin/virtualenv_test.sh
       - name: deploy
         env:
           WIKIBOT_TOKEN: ${{secrets.WIKIBOT_TOKEN}}

Dockerfile

@@ -29,7 +29,7 @@ RUN apt update ; \
 FROM twlight_base
 COPY --from=twlight_build /venv /venv
 COPY --from=quay.io/wikipedialibrary/debian_perl:latest /opt/perl /opt/perl
-ENV PATH="/opt/perl/bin:${PATH}" TWLIGHT_HOME=/app PYTHONUNBUFFERED=1 PYTHONPATH="/app:/venv"
+ENV PATH="/venv/bin:/opt/perl/bin:${PATH}" TWLIGHT_HOME=/app PYTHONUNBUFFERED=1 PYTHONPATH="/app:/venv"
 WORKDIR ${TWLIGHT_HOME}
 # Runtime dependencies.
 # Refactoring shell code could remove bash dependency
@@ -74,4 +74,4 @@ RUN SECRET_KEY=twlight /app/bin/twlight_static.sh
 
 EXPOSE 80
 
-ENTRYPOINT ["/app/bin/twlight_docker_entrypoint.sh"]
+ENTRYPOINT ["/app/bin/virtualenv_activate.sh"]

README.md

@@ -17,6 +17,3 @@ See the [Local setup](https://github.com/WikipediaLibrary/TWLight/wiki/Local-set
 Further project documentation is available in the [Wiki](https://github.com/WikipediaLibrary/TWLight/wiki). Our issue tracking takes place on the [Library-Card-Platform Phabricator](https://phabricator.wikimedia.org/project/board/2765/) board.
 
 [![Build Status](https://travis-ci.com/WikipediaLibrary/TWLight.svg?branch=master)](https://travis-ci.com/github/WikipediaLibrary/TWLight)
-
-[![Coveralls ](https://github.com/WikipediaLibrary/TWLight/actions/workflows/coveralls_check.yml/badge.svg)](https://github.com/WikipediaLibrary/TWLight/actions/workflows/coveralls_check.yml)
-[![Coverage Status](https://coveralls.io/repos/github/WikipediaLibrary/TWLight/badge.svg?branch=master)](https://coveralls.io/github/WikipediaLibrary/TWLight?branch=master)

TWLight/settings/local.py

@@ -48,6 +48,8 @@
                 "django.template.loaders.filesystem.Loader",
                 "django.template.loaders.app_directories.Loader",
             ],
+            # Coverage plugin requires debugging
+            "debug": True,
         },
     }
 ]

bin/debian_swarm_deploy.sh

@@ -102,7 +102,7 @@ echo "Deploying stack"
 docker stack deploy -c "docker-compose.yml" -c "docker-compose.${TWLIGHT_STACK_ENV}.yml" "${TWLIGHT_STACK_ENV}"
 echo "Setting up crontab. *WARNING* This will create duplicate jobs if run repeatedly."
 (crontab -l 2>/dev/null; echo "# Run django_cron tasks.") | crontab -
-(crontab -l 2>/dev/null; echo '*/5 * * * *  docker exec -t \$(docker ps -q -f name=${TWLIGHT_STACK_ENV}_twlight) /app/bin/twlight_docker_entrypoint.sh python manage.py runcrons') | crontab -
+(crontab -l 2>/dev/null; echo '*/5 * * * *  docker exec -t \$(docker ps -q -f name=${TWLIGHT_STACK_ENV}_twlight) python manage.py runcrons') | crontab -
 (crontab -l 2>/dev/null; echo "# Update the running TWLight service if there is a new image. The initial pull is just to verify that the image is valid. Otherwise an inaccessible image could break the service.") | crontab -
 (crontab -l 2>/dev/null; echo '*/5 * * * *  /srv/TWLight/bin/./twlight_docker_deploy.sh ${TWLIGHT_STACK_ENV} branch_${TWLIGHT_GIT_BRANCH}') | crontab -
 (crontab -l 2>/dev/null; echo "# Reclaim disk space previously used by docker.") | crontab -

bin/twlight_docker_entrypoint.sh

@@ -0,0 +1 @@
+bin/virtualenv_activate.sh

bin/twlight_restore.sh

@@ -19,18 +19,18 @@ restore_file=${1}
 tar -xvzf  "${restore_file}" -C "${TWLIGHT_HOME}" --no-overwrite-dir
 
 ## Import DB
-if ${TWLIGHT_HOME}/bin/virtualenv_wait_for_db.sh
+if "${TWLIGHT_HOME}/bin/virtualenv_wait_for_db.sh"
 then
-    ${TWLIGHT_HOME}/bin/twlight_mysqlimport.sh ${TWLIGHT_HOME}/twlight.sql
+    "${TWLIGHT_HOME}/bin/twlight_mysqlimport.sh" "${TWLIGHT_HOME}/twlight.sql"
 fi
 ## Don't leave an extra DB dump laying out.
 rm -f "${TWLIGHT_HOME}/twlight.sql"
 
 ## Set perms
-chown -R ${TWLIGHT_UNIXNAME} "${TWLIGHT_HOME}"
+chown -R "${TWLIGHT_UNIXNAME}" "${TWLIGHT_HOME}"
 find "${TWLIGHT_HOME}/media" -type f | xargs chmod 644
 
 ## Run any necessary DB operations.
-${TWLIGHT_HOME}/bin/virtualenv_migrate.sh
+"${TWLIGHT_HOME}/bin/virtualenv_migrate.sh"
 
 echo "Finished TWLight restore."

bin/virtualenv_activate.sh

@@ -11,7 +11,7 @@ fi
 # Get secrets.
 if  [ ! -n "${SECRET_KEY+isset}" ]
 then
-    source /app/bin/twlight_docker_secrets.sh
+    source bin/twlight_docker_secrets.sh
 fi
 
 # Virtualenv scripts need to be run as www
@@ -26,11 +26,11 @@ fi
 ACTIVATED=$(python3 -c 'import sys; print ("1" if hasattr(sys, "real_prefix") else "0")')
 if [ "${ACTIVATED}" -eq 0 ]
 then
-
     # Activate Django virtualenv.
     source /venv/bin/activate
 fi
 
 # Move to the project root.
-cd $TWLIGHT_HOME
+cd "$TWLIGHT_HOME" || exit 1
 
+exec "$@"

bin/virtualenv_test.sh

@@ -6,28 +6,27 @@ set -euo pipefail
 
 {
     # print the date for logging purposes
-    echo [$(date)]
+    echo "[$(date)]"
 
     # Load virtual environment
-    if source ${TWLIGHT_HOME}/bin/virtualenv_activate.sh
+    if source "${TWLIGHT_HOME}/bin/virtualenv_activate.sh"
     then
         # Run linter
         echo "black --target-version py311 --check TWLight"
         if black --target-version py311 --check TWLight
         then
             echo "${TWLIGHT_HOME}/tests/shunit/twlight_i18n_lint_test.sh"
-            ${TWLIGHT_HOME}/tests/shunit/twlight_i18n_lint_test.sh
+            "${TWLIGHT_HOME}/tests/shunit/twlight_i18n_lint_test.sh"
 
             # https://github.com/WikipediaLibrary/TWLight/wiki/Translation
             echo "Checking for localization issues"
-            # TODO: add html jinja template checks.
-            # find TWLight -type f \( -name "*.py" -o -name "*.html" \) -print0 | xargs -0 -I % ${TWLIGHT_HOME}/bin/twlight_i18n_lint.pl %
-            find TWLight -type f \( -name "*.py" \) -print0 | xargs -0 -I % ${TWLIGHT_HOME}/bin/twlight_i18n_lint.pl %
+            find TWLight -type f \( -name "*.py" \) -print0 | xargs -0 -I % "${TWLIGHT_HOME}/bin/twlight_i18n_lint.pl" %
             echo "No localization issues found"
 
             # Run test suite via coverage so we can get a report without having to run separate tests for it.
             DJANGO_LOG_LEVEL=CRITICAL DJANGO_SETTINGS_MODULE=TWLight.settings.local \
-            coverage run --source TWLight manage.py test --keepdb --noinput --parallel --timing
+            coverage run manage.py test --keepdb --noinput --parallel --timing
+			coverage report
         else
             # If linting fails, offer some useful feedback to the user.
             black --target-version py311 --quiet --diff TWLight