diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa56974..744f782 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,24 +3,22 @@ name: CI - Ubuntu on: push: branches: - - master + - main pull_request: - branches: - - master jobs: webtests: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.x architecture: x64 - - name: Checkout head - uses: actions/checkout@v2 + - name: Checkout head + uses: actions/checkout@v4 - name: Run webtests - run: chmod +x ./run_web_tests.sh && ./run_web_tests.sh + run: bash ./run_web_tests.sh env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -28,16 +26,15 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.x architecture: x64 - - name: Checkout head - uses: actions/checkout@v2 + - name: Checkout head + uses: actions/checkout@v4 - name: Install flake8 run: pip install flake8 - name: Run flake8 run: cd web && flake8 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.gitignore b/.gitignore index 3dd8f2d..39c78fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,171 @@ +# Environment files .env + +# Shared data directories for containers pgdata/ webdata/ + +# Docker override files for local development options +docker-compose.override.yml + +# Stanadard Python gitignore +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.test_env b/.test_env deleted file mode 100644 index 0162fe3..0000000 --- a/.test_env +++ /dev/null @@ -1,4 +0,0 @@ -DB_NAME=TEST -DB_USER=TESTUSER -DB_PASS=password -HOST_PORT=8082 diff --git a/DevelopmentSetup.md b/DevelopmentSetup.md index 896d64b..3006051 100644 --- a/DevelopmentSetup.md +++ b/DevelopmentSetup.md @@ -206,11 +206,32 @@ Not removing this data will prevent you from running the Django Admin Account st sudo rm -rf pgdata/ ``` -## Troubleshooting +## Enable live code editing + +By default changes to the Python code are not reflected in the running version until +`bin/shutdown.sh` followed by `bin/boot.sh`. This is because the Python in `web` +is copied inside the Docker image at build time. + +It is possible to configure the setup locally such that edits to the Python files +in `web` are shared with the running containers immediately. Add a file, alongside +`docker-compose.yml`, called `docker-compose.override.yml` with the following contents: + +```yml +version: '3' + +services: + web: + environment: + - DEBUG=true + volumes: + - ./web:/usr/src/app + - webdata:/static +``` -If you are having problems connecting to the localhost with an obscure error code, try -add the ``DEBUG=True`` variable to your ``.env`` file. This will provide a better -error message to help you debug the problem. +This file is automatically registered by `docker-compose` and overrides the default +production configuration. **Do not add this file to the version control system!**. + +## Troubleshooting An error about invalid credentials could be caused by persisting data from a previous time you have looked at this repo. To clear any persisting data, run the shutdown script. diff --git a/bin/boot.sh b/bin/boot.sh index 7f9e862..347abdd 100755 --- a/bin/boot.sh +++ b/bin/boot.sh @@ -18,11 +18,27 @@ fi # Create empty directories, so they are not owned by root mkdir -p "$SOURCE_DIR/pgdata" -mkdir -p "$SOURCE_DIR/webdata" # Start external network create_external_net # Build services docker-compose --project-name ${PROJECT_NAME} build # Bring up the stack and detach -docker-compose --project-name ${PROJECT_NAME} up -d +docker-compose --project-name ${PROJECT_NAME} up --detach + +# Replace this loop with `--wait-timeout` when it is available +# with 'docker compose' v2 +sleep_counter=0 +while [ $(docker-compose ps --quiet --filter status=running | wc -l | xargs) -ne 4 ] +do + sleep 1 + let counter++ + if [[ $counter -eq 30 ]]; then + echo "Services have not come up to a running state." + exit 1 + fi + echo "Waiting for all services to be running." +done + + +echo "Services up and running." diff --git a/bin/shutdown.sh b/bin/shutdown.sh index fcb21d6..473d5ee 100755 --- a/bin/shutdown.sh +++ b/bin/shutdown.sh @@ -12,8 +12,7 @@ docker-compose down # the web data volume shouldn't really be persistent as all of the files # come from an image echo "Removing webdata volume so it is rebuilt on next startup" -rm -r "${SOURCE_DIR}/webdata" -docker volume rm errorreports_webdata +docker volume rm ${PROJECT_NAME}_webdata echo "Removing external network nginx_net" docker network rm nginx_net diff --git a/blank.env b/blank.env index c129fe1..30fa9dd 100644 --- a/blank.env +++ b/blank.env @@ -1,20 +1,17 @@ -# Default config -DB_NAME=django +DEBUG=false +HOST_PORT=8083 +# Django settings. Passwords are in the password management tool +DB_NAME=django # Generate with pwgen -s 32 1 SECRET_KEY= - -# These are located in the password management tool DB_USER= DB_PASS= - DB_SERVICE=postgres DB_PORT=5432 # Can be found Slack settings SLACK_WEBHOOK_URL= - -HOST_PORT=8083 SLACK_ERROR_REPORTS_CHANNEL=#error-reports SLACK_ERROR_REPORTS_USERNAME="Error Reporter" SLACK_ERROR_REPORTS_EMOJI=:sadmantid: diff --git a/docker-compose.yml b/docker-compose.yml index 2155ffa..ddad42e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: restart: always build: ./web volumes: - - webdata:/usr/src/app + - webdata:/static expose: # Change the value in the nginx configuration if this is changed - "8000" @@ -46,9 +46,6 @@ services: SECRET_KEY: ${SECRET_KEY} # Define this in .env for development mode. DO NOT USE IN PRODUCTION DEBUG: ${DEBUG} - MAIL_PASS: ${MAIL_PASS} - MAIL_PORT: ${MAIL_PORT} - ERROR_EMAIL: ${ERROR_EMAIL} nginx-errorreports: restart: always @@ -57,7 +54,7 @@ services: ports: - "${HOST_PORT}:80" volumes: - - webdata:/usr/src/app + - webdata:/static networks: - default - nginx_net @@ -66,6 +63,7 @@ volumes: # This allows Nginx to serve static content, as Django won't... webdata: + networks: nginx_net: external: true diff --git a/nginx/sites-enabled/errorreports b/nginx/sites-enabled/errorreports index 7f78e1f..2693f4d 100644 --- a/nginx/sites-enabled/errorreports +++ b/nginx/sites-enabled/errorreports @@ -16,7 +16,7 @@ server { } location /static { - alias /usr/src/app/static; + root /; } location / { diff --git a/run_web_tests.sh b/run_web_tests.sh index 13fa382..1af8703 100644 --- a/run_web_tests.sh +++ b/run_web_tests.sh @@ -1,4 +1,48 @@ -cp .test_env .env -./bin/boot.sh -sleep 30 -docker exec -t errorreports_web_1 sh -c "python manage.py test" \ No newline at end of file +ENV_FILE=.env +DB_DIR=pgdata +DJANGO_SERVICE_NAME=web +CLEAN_ENV=false +CLEAN_SERVICES=false + +if [[ ! -f "$ENV_FILE" ]]; then + if [[ -d ${DB_DIR} ]]; then + echo "A $ENV_FILE file could not be found yet a '${DB_DIR}' directory exists." + echo "This implies a database has been created but the credentials are missing." + echo "Tests will fail as they won't be able to access the database." + echo "IF IN A DEVELOPMENT ENVIRONMENT remove ${DB_DIR} and re-run this script." + exit 1 + fi + + echo "No $ENV_FILE file and no DB found. Generating .env file for tests." + echo "Both .env file and DB will be cleaned " + cat blank.env |\ + sed -e "s@DEBUG=false@DEBUG=true@" |\ + sed -e "s@SECRET_KEY=@SECRET_KEY=123456789abcdefghijkl@" |\ + sed -e 's@DB_USER=@DB_USER=testuser@' |\ + sed -e 's@DB_PASS=@DB_PASS=testuserpasswd@' |\ + sed -e 's@SLACK_WEBHOOK_URL=@SLACK_WEBHOOK_URL=@' > ${ENV_FILE} + CLEAN_ENV=true +else + echo "Running tests using an existing environment file..." +fi + +if [[ -z $(docker-compose ps --quiet --filter status=running $DJANGO_SERVICE_NAME) ]]; then + echo "Booting services." + ./bin/boot.sh + CLEAN_SERVICES=true +else + echo "Services already running.." +fi + +echo "Executing web tests..." +docker-compose exec $DJANGO_SERVICE_NAME sh -c "python manage.py test $@" + +# Clean up +if [[ $CLEAN_SERVICES == true ]]; then + echo "Shutting down services that were auto started." + ./bin/shutdown.sh +fi +if [[ $CLEAN_ENV == true ]]; then + echo "Cleaning auto-generated .env file" + rm -f $ENV_FILE +fi diff --git a/web/run_django.sh b/web/run_django.sh index 9deefe7..2eda878 100755 --- a/web/run_django.sh +++ b/web/run_django.sh @@ -3,8 +3,6 @@ # Collect static files into location shared by nginx echo "Running collectstatic..." python manage.py collectstatic --noinput -echo "Fixing permissions on /usr/src/app/static" -chmod 755 -R /usr/src/app/static # Create any new DB migrations and apply those in version control. # Note that running this script for the very first time produces @@ -20,9 +18,9 @@ python manage.py makemigrations --noinput echo "Running migrate..." python manage.py migrate --noinput -# If running in DEBUG mode add debug logging to gunicorn -if [ -n "${DEBUG}" ]; then - DEBUG_ARGS="--log-level debug --capture-output" +# If running in DEBUG mode add debug logging and hot reload to gunicorn +if [[ ${DEBUG} == true ]]; then + DEBUG_ARGS="--log-level debug --capture-output --reload" else DEBUG_ARGS= fi diff --git a/web/services/admin.py b/web/services/admin.py index 4e69d70..0abd0db 100644 --- a/web/services/admin.py +++ b/web/services/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin from django.urls import reverse -from django.utils.safestring import mark_safe +from django.utils.safestring import mark_safe # Register your models here. from services.models import ErrorReport, UserDetails diff --git a/web/services/tasks.py b/web/services/tasks.py index 55fc690..4949956 100644 --- a/web/services/tasks.py +++ b/web/services/tasks.py @@ -22,7 +22,7 @@ def send_notification_to_slack(name, :param additional_text: Any additional text provided """ slack_webhook_url = settings.SLACK_WEBHOOK_URL - if slack_webhook_url is None: + if not slack_webhook_url: return text = """Name: {} Email: {} Additional text: diff --git a/web/settings.py b/web/settings.py index 3c1e9f4..f1085b4 100644 --- a/web/settings.py +++ b/web/settings.py @@ -23,9 +23,9 @@ SECRET_KEY = os.environ['SECRET_KEY'] # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.getenv('DEBUG', 'NO').lower() in ('on', 'true', 'y', 'yes') +DEBUG = (os.getenv('DEBUG', 'false').lower() == 'true') -DEFAULT_AUTO_FIELD='django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' ALLOWED_HOSTS = ['*'] @@ -120,13 +120,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.8/howto/static-files/ - STATIC_URL = '/static/' -# STATICFILES_DIRS = ( -# os.path.join(BASE_DIR, 'static'), -# ) -# print(STATICFILES_DIRS) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_ROOT = '/static' # Logging if DEBUG: diff --git a/web/settings.pyc b/web/settings.pyc deleted file mode 100644 index 0a60ce0..0000000 Binary files a/web/settings.pyc and /dev/null differ