From 7a8b5ba7f7a236af450d94b701759b789dd5842a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 7 Jan 2026 16:11:57 -0800 Subject: [PATCH 01/98] wip: use django-async-upload for media records --- TEKDB/TEKDB/models.py | 4 +++- TEKDB/TEKDB/settings.py | 1 + TEKDB/TEKDB/urls.py | 1 + TEKDB/requirements.txt | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index a4dd8d3a..4a31117f 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -24,6 +24,8 @@ from django.conf import settings from django.contrib.gis.db.models import GeometryField from tinymce.models import HTMLField +from admin_async_upload.models import AsyncFileField + # from moderation.db import ModeratedModel import os @@ -2753,7 +2755,7 @@ class Media(Reviewable, Queryable, Record, ModeratedModel): null=True, verbose_name="historic location", ) - mediafile = models.FileField( + mediafile = AsyncFileField( db_column="mediafile", max_length=255, blank=True, diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 1b0a3d10..7be9ba58 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -70,6 +70,7 @@ "Relationships", "reversion", "django.contrib.sites", + "admin_async_upload", # 'moderation.apps.SimpleModerationConfig', ] diff --git a/TEKDB/TEKDB/urls.py b/TEKDB/TEKDB/urls.py index ef2e60f3..536e2d27 100644 --- a/TEKDB/TEKDB/urls.py +++ b/TEKDB/TEKDB/urls.py @@ -77,6 +77,7 @@ views.ResourceActivityAutocompleteView.as_view(), name="select2_fk_resourceactivity", ), + re_path(r"^admin_async_upload/", include("admin_async_upload.urls")), path("", include("explore.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 276851e8..590b2934 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -14,6 +14,7 @@ pillow psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 +django-async-upload XlsxWriter #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation From 619adf59d34ea12f84271bb8aada2f9e33e0e2f9 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 8 Jan 2026 15:14:40 -0800 Subject: [PATCH 02/98] use local admin async libraary during development --- TEKDB/Dockerfile | 1 + TEKDB/entrypoint.sh | 8 ++++++++ TEKDB/requirements.txt | 4 +--- docker/docker-compose.yml | 1 + 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index 896b7676..1ba7ac1f 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -22,6 +22,7 @@ WORKDIR /usr/src/app COPY requirements.txt requirements_linux.txt /usr/src/app/ # Upgrade pip and install Python dependencies +# Note: editable packages (-e) will be installed at runtime via entrypoint.sh RUN pip install --upgrade pip \ && pip install -r requirements.txt -r requirements_linux.txt diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index fac43bbe..fc9e8b00 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -3,6 +3,14 @@ # Exit on errors set -e +# Install editable package if the directory is mounted +if [ -d "/django-resumable-async-upload" ]; then + echo "Installing django-resumable-async-upload in editable mode for local development..." + pip install -e /django-resumable-async-upload +else + echo "django-resumable-async-upload directory not found, skipping editable install" +fi + # If a SQL_HOST is provided, wait for Postgres to become available before running # migrations. This prevents race conditions when using docker-compose where the # web container starts before the DB is ready. diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 590b2934..b7afcd26 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -14,8 +14,6 @@ pillow psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 -django-async-upload - XlsxWriter #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation @@ -36,4 +34,4 @@ ipython # Serve static files in production containers whitenoise>=6.0.0,<7.0.0 # FORMATTING AND LINTING -ruff +ruff \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f865e83a..0db8d15d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,6 +40,7 @@ services: - "8000:8000" volumes: - ../TEKDB:/usr/src/app + - ../../django-resumable-async-upload:/django-resumable-async-upload volumes: tekdb_db_data: From 9546b7625f500475bdaf22482642c2825c51931f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 13 Jan 2026 14:22:08 -0800 Subject: [PATCH 03/98] use AsyncFileCleanupMixin in MediaAdmin form --- TEKDB/TEKDB/admin.py | 4 +++- TEKDB/TEKDB/settings.py | 3 +++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 04c5bbb5..c6eeb413 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -60,6 +60,8 @@ from TEKDB.settings import ADMIN_SITE_HEADER from TEKDB.settings import BASE_DIR from TEKDB.widgets import OpenLayers6Widget +from admin_async_upload.admin import AsyncFileCleanupMixin + admin.site.site_header = ADMIN_SITE_HEADER @@ -830,7 +832,7 @@ def has_add_permission(self, request): @admin.register(Media) -class MediaAdmin(RecordAdminProxy, RecordModelAdmin): +class MediaAdmin(AsyncFileCleanupMixin, RecordAdminProxy, RecordModelAdmin): readonly_fields = ( "medialink", "enteredbyname", diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 7be9ba58..a003c68d 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -83,6 +83,7 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + "admin_async_upload.middleware.OrphanedFileCleanupMiddleware", ] ROOT_URLCONF = "TEKDB.urls" @@ -347,6 +348,8 @@ DEFAULT_MAXIMUM_RESULTS = 500 +ADMIN_RESUMABLE_SHOW_THUMB = True + try: from TEKDB.local_settings import * # noqa: F403 except Exception: From 630ae9694fb1792b106b2de18c0529b580d2f7eb Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 20 Jan 2026 11:58:25 -0800 Subject: [PATCH 04/98] support async uploads for MediaBulkUploads --- TEKDB/TEKDB/admin.py | 117 ++++++++++++++++++++++++------------------ TEKDB/TEKDB/forms.py | 12 ++++- TEKDB/TEKDB/models.py | 1 + 3 files changed, 78 insertions(+), 52 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index c6eeb413..885cfee4 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -615,7 +615,7 @@ class CitationsAdmin(RecordAdminProxy, RecordModelAdmin): # * Bulk Media Upload Admin @admin.register(MediaBulkUpload) -class MediaBulkUploadAdmin(admin.ModelAdmin): +class MediaBulkUploadAdmin(AsyncFileCleanupMixin, admin.ModelAdmin): form = MediaBulkUploadForm list_display = ("mediabulkname", "mediabulkdate", "enteredbyname", "enteredbydate") @@ -628,57 +628,74 @@ def save_model(self, request, obj, form, change): activities = form.cleaned_data.get("activities") placesresources = form.cleaned_data.get("placesresources") - for file in request.FILES.getlist("files"): - mime_type, _ = guess_type(file.name) - # if mime_type: - file_mime_type = mime_type.split("/")[0] - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith=file_mime_type - ).first() - if media_type_instance: - mediatype = media_type_instance - else: - media_type_instance = LookupMediaType.objects.filter( - mediatype__startswith="other" - ).first() + # Handle async uploaded file (comes as a comma separated string of file paths) + uploaded_file_paths = form.cleaned_data.get("files") + if uploaded_file_paths: + from django.core.files.storage import default_storage + import os + + # split the comma-separated string into a list + # TODO: need to add a step in the frontend to clean up any commas in filenames + uploaded_file_paths_list = uploaded_file_paths.split(",") + + for uploaded_file_path in uploaded_file_paths_list: + # Extract just the filename from the path + file_name = os.path.basename(uploaded_file_path) + + # Guess MIME type from filename + mime_type, _ = guess_type(file_name) + if mime_type: + file_mime_type = mime_type.split("/")[0] + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith=file_mime_type + ).first() + else: + media_type_instance = None + + if not media_type_instance: + media_type_instance = LookupMediaType.objects.filter( + mediatype__startswith="other" + ).first() + mediatype = media_type_instance - filename = file.name.split(".")[0] + filename = file_name.rsplit(".", 1)[0] # Remove extension - media_instance = Media( - medianame=filename, - mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', - mediafile=file, - mediatype=mediatype, - ) - media_instance.save() - obj.mediabulkupload.add(media_instance) - - # Add relationships - if places: - for place in places: - PlacesMediaEvents.objects.create( - placeid=place, mediaid=media_instance - ) - if resources: - for resource in resources: - ResourcesMediaEvents.objects.create( - resourceid=resource, mediaid=media_instance - ) - if citations: - for citation in citations: - MediaCitationEvents.objects.create( - citationid=citation, mediaid=media_instance - ) - if activities: - for activity in activities: - ResourceActivityMediaEvents.objects.create( - resourceactivityid=activity, mediaid=media_instance - ) - if placesresources: - for placeresource in placesresources: - PlacesResourceMediaEvents.objects.create( - placeresourceid=placeresource, mediaid=media_instance - ) + media_instance = Media( + medianame=filename, + mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}', + mediafile=uploaded_file_path, + mediatype=mediatype, + ) + print(f"[DEBUG]: Creating Media instance: {filename}") + media_instance.save() + obj.mediabulkupload.add(media_instance) + + # Add relationships + if places: + for place in places: + PlacesMediaEvents.objects.create( + placeid=place, mediaid=media_instance + ) + if resources: + for resource in resources: + ResourcesMediaEvents.objects.create( + resourceid=resource, mediaid=media_instance + ) + if citations: + for citation in citations: + MediaCitationEvents.objects.create( + citationid=citation, mediaid=media_instance + ) + if activities: + for activity in activities: + ResourceActivityMediaEvents.objects.create( + resourceactivityid=activity, mediaid=media_instance + ) + if placesresources: + for placeresource in placesresources: + PlacesResourceMediaEvents.objects.create( + placeresourceid=placeresource, mediaid=media_instance + ) @admin.display(description="Thumbnails") def thumbnail_gallery(self, obj): diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 9808785c..78d52cb1 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,5 +1,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple +from admin_async_upload.fields import FormResumableFileField +from admin_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, Places, @@ -7,6 +9,7 @@ Citations, ResourcesActivityEvents, PlacesResourceEvents, + Media, ) from .widgets import ThumbnailFileInput @@ -26,7 +29,13 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - files = MultipleFileField() + # files = MultipleFileField() + files = FormResumableFileField( + required=False, + # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), + # which is what we want for bulk upload. + widget=ResumableAdminWidget(attrs={"model": Media, "field_name": "mediafile"}), + ) places = forms.ModelMultipleChoiceField( queryset=Places.objects.all(), required=False, @@ -59,7 +68,6 @@ class Meta: "mediabulkname", "mediabulkdescription", "mediabulkdate", - "files", "places", "resources", "citations", diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index 4a31117f..1eada9a2 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -2761,6 +2761,7 @@ class Media(Reviewable, Queryable, Record, ModeratedModel): blank=True, null=True, verbose_name="file", + max_files=1, ) limitedaccess = models.BooleanField( db_column="limitedaccess", From 2484eae951ae200a66b95177d976209c8c50dda8 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 21 Jan 2026 15:00:12 -0800 Subject: [PATCH 05/98] fix tests for media bulk upload admin --- TEKDB/TEKDB/admin.py | 2 +- TEKDB/TEKDB/tests/test_admin.py | 61 ++++++--------------------------- 2 files changed, 11 insertions(+), 52 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 885cfee4..95d84a1b 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -630,8 +630,8 @@ def save_model(self, request, obj, form, change): # Handle async uploaded file (comes as a comma separated string of file paths) uploaded_file_paths = form.cleaned_data.get("files") + if uploaded_file_paths: - from django.core.files.storage import default_storage import os # split the comma-separated string into a list diff --git a/TEKDB/TEKDB/tests/test_admin.py b/TEKDB/TEKDB/tests/test_admin.py index 8a713b86..bbf28f63 100644 --- a/TEKDB/TEKDB/tests/test_admin.py +++ b/TEKDB/TEKDB/tests/test_admin.py @@ -1,7 +1,6 @@ # from django.conf import settings from django.test import RequestFactory from unittest.mock import patch -from django.core.files.uploadedfile import SimpleUploadedFile from django.contrib.auth import get_user_model from django.contrib.admin.sites import AdminSite from django.urls import reverse @@ -165,9 +164,8 @@ def test_media_bulk_upload_admin_add(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) + + test_image_paths = "test_image.jpg,test_image.jpg" place = Places.objects.create(indigenousplacename="Test Place") resource = Resources.objects.create(commonname="Test Resource") @@ -180,7 +178,7 @@ def test_media_bulk_upload_admin_add(self): activity = ResourcesActivityEvents.objects.create(placeresourceid=placeresource) post_data = { - "files": [test_image, test_image], + "files": test_image_paths, "places": [place.pk], "resources": [resource.pk], "citations": [citation.pk], @@ -208,14 +206,6 @@ def test_media_bulk_upload_admin_add(self): PlacesResourceEvents.objects.filter(pk=placeresource.pk).exists() ) - for media in Media.objects.filter(medianame="test_image"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() - # Clean up related objects activity.delete() placeresource.delete() @@ -227,14 +217,11 @@ def test_media_bulk_upload_admin_other_types(self): from TEKDB.admin import MediaBulkUploadAdmin url = reverse("admin:TEKDB_mediabulkupload_add") - test_other_type = SimpleUploadedFile( - "./test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) request = self.factory.post( url, { - "files": [test_other_type, test_other_type], + "files": "test_thing.shp,test_thing.shp", }, ) @@ -245,48 +232,20 @@ def test_media_bulk_upload_admin_other_types(self): bulk_admin.save_model( obj=MediaBulkUpload(), request=request, form=bulk_form, change=None ) - - for media in Media.objects.filter(medianame="test_thing"): - self.assertTrue(os.path.exists(media.mediafile.path)) - os.remove( - media.mediafile.path - ) # Clean up the uploaded files after the test - self.assertFalse(os.path.exists(media.mediafile.path)) - media.delete() + self.assertTrue(Media.objects.filter(medianame="test_thing").exists()) + self.assertEqual(Media.objects.filter(medianame="test_thing").count(), 2) def test_media_bulk_upload_admin_thumbnail_gallery(self): from TEKDB.admin import MediaBulkUploadAdmin from TEKDB.models import MediaBulkUpload, Media url = reverse("admin:TEKDB_mediabulkupload_add") - test_image = SimpleUploadedFile( - "./thumbnail_test_image.jpg", b"\x00\x00\x00\x00", content_type="image" - ) - test_video = SimpleUploadedFile( - "./thumbnail_test_video.mp4", b"\x00\x00\x00\x00", content_type="video" - ) - test_audio = SimpleUploadedFile( - "./thumbnail_test_audio.mp3", b"\x00\x00\x00\x00", content_type="audio" - ) - test_text = SimpleUploadedFile( - "./thumbnail_test_text.txt", b"\x00\x00\x00\x00", content_type="text" - ) - test_other = SimpleUploadedFile( - "./thumbnail_test_thing.shp", b"\x00\x00\x00\x00", content_type="other" - ) - test_unknown_type = SimpleUploadedFile( - "./thumbnail_test_unknown.xyz", b"\x00\x00\x00\x00", content_type="unknown" - ) + test_file_paths = [ + "thumbnail_test_image.jpg, thumbnail_test_video.mp4, thumbnail_test_audio.mp3, thumbnail_test_text.txt, thumbnail_test_thing.shp, thumbnail_test_unknown.xyz" + ] post_data = { - "files": [ - test_image, - test_video, - test_audio, - test_text, - test_other, - test_unknown_type, - ], + "files": test_file_paths, } request = self.factory.post(url, post_data) From 56004dc08799c39b6a466021ce578f20686298f6 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 23 Jan 2026 14:09:38 -0800 Subject: [PATCH 06/98] set ADMIN_SIMULTANEOUS_UPLOADS to 1 --- TEKDB/TEKDB/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index a003c68d..2b759a9f 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -348,7 +348,9 @@ DEFAULT_MAXIMUM_RESULTS = 500 +# Django Resumable Async Upload settings ADMIN_RESUMABLE_SHOW_THUMB = True +ADMIN_SIMULTANEOUS_UPLOADS = 1 try: from TEKDB.local_settings import * # noqa: F403 From d251595c594dd0333b93cab992e139c8336572ff Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 23 Jan 2026 16:42:17 -0800 Subject: [PATCH 07/98] use ADMIN_RESUMABLE_CHUNK_FOLDER setting --- TEKDB/TEKDB/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 2b759a9f..9fb2626a 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -351,6 +351,7 @@ # Django Resumable Async Upload settings ADMIN_RESUMABLE_SHOW_THUMB = True ADMIN_SIMULTANEOUS_UPLOADS = 1 +ADMIN_RESUMABLE_CHUNK_FOLDER = "resumable_chunks" try: from TEKDB.local_settings import * # noqa: F403 From 22a25afb409cd207b845d25b1943de9a6cc3f838 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 26 Jan 2026 13:45:27 -0800 Subject: [PATCH 08/98] clean up comments --- TEKDB/TEKDB/admin.py | 1 - TEKDB/TEKDB/forms.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index 95d84a1b..c3307512 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -635,7 +635,6 @@ def save_model(self, request, obj, form, change): import os # split the comma-separated string into a list - # TODO: need to add a step in the frontend to clean up any commas in filenames uploaded_file_paths_list = uploaded_file_paths.split(",") for uploaded_file_path in uploaded_file_paths_list: diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 78d52cb1..3f236fb9 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -29,7 +29,6 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - # files = MultipleFileField() files = FormResumableFileField( required=False, # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), @@ -68,6 +67,7 @@ class Meta: "mediabulkname", "mediabulkdescription", "mediabulkdate", + "files", "places", "resources", "citations", From a0eed17a96c6989883abc7ce7b4c06c054d13f34 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 27 Jan 2026 09:32:18 -0800 Subject: [PATCH 09/98] remove debug log --- TEKDB/TEKDB/admin.py | 1 - 1 file changed, 1 deletion(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index c3307512..d6785deb 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -665,7 +665,6 @@ def save_model(self, request, obj, form, change): mediafile=uploaded_file_path, mediatype=mediatype, ) - print(f"[DEBUG]: Creating Media instance: {filename}") media_instance.save() obj.mediabulkupload.add(media_instance) From 2671ad760f37c0da15707e28bbc0619732826336 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 27 Jan 2026 15:22:36 -0800 Subject: [PATCH 10/98] remove OrphanedFileCleanupMiddleware and AsyncFileCleanupMixin --- TEKDB/TEKDB/admin.py | 5 ++--- TEKDB/TEKDB/settings.py | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index d6785deb..ee17bf84 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -60,7 +60,6 @@ from TEKDB.settings import ADMIN_SITE_HEADER from TEKDB.settings import BASE_DIR from TEKDB.widgets import OpenLayers6Widget -from admin_async_upload.admin import AsyncFileCleanupMixin admin.site.site_header = ADMIN_SITE_HEADER @@ -615,7 +614,7 @@ class CitationsAdmin(RecordAdminProxy, RecordModelAdmin): # * Bulk Media Upload Admin @admin.register(MediaBulkUpload) -class MediaBulkUploadAdmin(AsyncFileCleanupMixin, admin.ModelAdmin): +class MediaBulkUploadAdmin(admin.ModelAdmin): form = MediaBulkUploadForm list_display = ("mediabulkname", "mediabulkdate", "enteredbyname", "enteredbydate") @@ -847,7 +846,7 @@ def has_add_permission(self, request): @admin.register(Media) -class MediaAdmin(AsyncFileCleanupMixin, RecordAdminProxy, RecordModelAdmin): +class MediaAdmin(RecordAdminProxy, RecordModelAdmin): readonly_fields = ( "medialink", "enteredbyname", diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 9fb2626a..e12a2e25 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -83,7 +83,6 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "admin_async_upload.middleware.OrphanedFileCleanupMiddleware", ] ROOT_URLCONF = "TEKDB.urls" From 736938caf2a54568b1905b9001af8de6a8375721 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 28 Jan 2026 11:47:15 -0800 Subject: [PATCH 11/98] refactor MediaBulkUploadForm to use FormResumableMultipleFileField --- TEKDB/TEKDB/admin.py | 9 +++------ TEKDB/TEKDB/forms.py | 4 ++-- TEKDB/TEKDB/tests/test_admin.py | 16 +++++++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index ee17bf84..cc551409 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -627,16 +627,13 @@ def save_model(self, request, obj, form, change): activities = form.cleaned_data.get("activities") placesresources = form.cleaned_data.get("placesresources") - # Handle async uploaded file (comes as a comma separated string of file paths) + # Handle async uploaded file (comes as a list of file path strings) uploaded_file_paths = form.cleaned_data.get("files") - if uploaded_file_paths: + if uploaded_file_paths and isinstance(uploaded_file_paths, list): import os - # split the comma-separated string into a list - uploaded_file_paths_list = uploaded_file_paths.split(",") - - for uploaded_file_path in uploaded_file_paths_list: + for uploaded_file_path in uploaded_file_paths: # Extract just the filename from the path file_name = os.path.basename(uploaded_file_path) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index 3f236fb9..d9319797 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,6 +1,6 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple -from admin_async_upload.fields import FormResumableFileField +from admin_async_upload.fields import FormResumableMultipleFileField from admin_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, @@ -29,7 +29,7 @@ def clean(self, data, initial=None): class MediaBulkUploadForm(forms.ModelForm): - files = FormResumableFileField( + files = FormResumableMultipleFileField( required=False, # not passing max_files here because FormResumableFileField defaults to undefined (unlimited), # which is what we want for bulk upload. diff --git a/TEKDB/TEKDB/tests/test_admin.py b/TEKDB/TEKDB/tests/test_admin.py index bbf28f63..b433fdb4 100644 --- a/TEKDB/TEKDB/tests/test_admin.py +++ b/TEKDB/TEKDB/tests/test_admin.py @@ -1,4 +1,5 @@ # from django.conf import settings +import json from django.test import RequestFactory from unittest.mock import patch from django.contrib.auth import get_user_model @@ -165,7 +166,7 @@ def test_media_bulk_upload_admin_add(self): url = reverse("admin:TEKDB_mediabulkupload_add") - test_image_paths = "test_image.jpg,test_image.jpg" + test_image_paths = ["test_image.jpg", "test_image.jpg"] place = Places.objects.create(indigenousplacename="Test Place") resource = Resources.objects.create(commonname="Test Resource") @@ -178,7 +179,7 @@ def test_media_bulk_upload_admin_add(self): activity = ResourcesActivityEvents.objects.create(placeresourceid=placeresource) post_data = { - "files": test_image_paths, + "files": json.dumps(test_image_paths), "places": [place.pk], "resources": [resource.pk], "citations": [citation.pk], @@ -221,7 +222,7 @@ def test_media_bulk_upload_admin_other_types(self): request = self.factory.post( url, { - "files": "test_thing.shp,test_thing.shp", + "files": json.dumps(["test_thing.shp", "test_thing.shp"]), }, ) @@ -241,11 +242,16 @@ def test_media_bulk_upload_admin_thumbnail_gallery(self): url = reverse("admin:TEKDB_mediabulkupload_add") test_file_paths = [ - "thumbnail_test_image.jpg, thumbnail_test_video.mp4, thumbnail_test_audio.mp3, thumbnail_test_text.txt, thumbnail_test_thing.shp, thumbnail_test_unknown.xyz" + "thumbnail_test_image.jpg", + "thumbnail_test_video.mp4", + "thumbnail_test_audio.mp3", + "thumbnail_test_text.txt", + "thumbnail_test_thing.shp", + "thumbnail_test_unknown.xyz", ] post_data = { - "files": test_file_paths, + "files": json.dumps(test_file_paths), } request = self.factory.post(url, post_data) From 0788466f7d38a654cf387fdb833641a595703cb7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 28 Jan 2026 11:49:18 -0800 Subject: [PATCH 12/98] remove unused ThumbnailFileInput and MultipleFileField --- TEKDB/TEKDB/forms.py | 15 --------------- TEKDB/TEKDB/widgets.py | 22 ---------------------- 2 files changed, 37 deletions(-) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index d9319797..be04eeb0 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -11,21 +11,6 @@ PlacesResourceEvents, Media, ) -from .widgets import ThumbnailFileInput - - -class MultipleFileField(forms.FileField): - def __init__(self, *args, **kwargs): - kwargs.setdefault("widget", ThumbnailFileInput) - super().__init__(*args, **kwargs) - - def clean(self, data, initial=None): - single_file_clean = super().clean - if isinstance(data, (list, tuple)): - result = [single_file_clean(d, initial) for d in data] - else: - result = [single_file_clean(data, initial)] - return result class MediaBulkUploadForm(forms.ModelForm): diff --git a/TEKDB/TEKDB/widgets.py b/TEKDB/TEKDB/widgets.py index 200fb1f0..4187a729 100644 --- a/TEKDB/TEKDB/widgets.py +++ b/TEKDB/TEKDB/widgets.py @@ -2,8 +2,6 @@ from django.contrib.gis.geometry import json_regex from django.contrib.gis.forms.widgets import BaseGeometryWidget -from django.forms.widgets import ClearableFileInput -from django.utils.safestring import mark_safe logger = logging.getLogger("django.contrib.gis") @@ -61,23 +59,3 @@ class Media: "assets/openlayers6/ol.js", "gis/js/OL6MapPolygonWidget.js", ) - - -class ThumbnailFileInput(ClearableFileInput): - template_name = "widgets/thumbnail_file_input.html" - allow_multiple_selected = True # Enable multiple file uploads - - def format_value(self, value): - if value and hasattr(value, "url"): - return mark_safe(f'') - return super().format_value(value) - - def __init__(self, attrs=None): - if attrs is None: - attrs = {} - attrs.update( - { - "multiple": "multiple" # Allow multiple file selection - } - ) - super().__init__(attrs) From 95da7bf87db6612639c2722ed32c962767a23712 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 10:38:36 -0800 Subject: [PATCH 13/98] use django_resumable_async_upload package --- TEKDB/TEKDB/forms.py | 4 ++-- TEKDB/TEKDB/models.py | 2 +- TEKDB/TEKDB/settings.py | 2 +- TEKDB/TEKDB/urls.py | 5 ++++- TEKDB/entrypoint.sh | 8 -------- TEKDB/requirements.txt | 1 + docker/docker-compose.yml | 1 - 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/TEKDB/TEKDB/forms.py b/TEKDB/TEKDB/forms.py index be04eeb0..1be8e12a 100644 --- a/TEKDB/TEKDB/forms.py +++ b/TEKDB/TEKDB/forms.py @@ -1,7 +1,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple -from admin_async_upload.fields import FormResumableMultipleFileField -from admin_async_upload.widgets import ResumableAdminWidget +from django_resumable_async_upload.fields import FormResumableMultipleFileField +from django_resumable_async_upload.widgets import ResumableAdminWidget from .models import ( MediaBulkUpload, Places, diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index 1eada9a2..f91e20fb 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -24,7 +24,7 @@ from django.conf import settings from django.contrib.gis.db.models import GeometryField from tinymce.models import HTMLField -from admin_async_upload.models import AsyncFileField +from django_resumable_async_upload.models import AsyncFileField # from moderation.db import ModeratedModel diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index e12a2e25..888e00a6 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -70,7 +70,7 @@ "Relationships", "reversion", "django.contrib.sites", - "admin_async_upload", + "django_resumable_async_upload", # 'moderation.apps.SimpleModerationConfig', ] diff --git a/TEKDB/TEKDB/urls.py b/TEKDB/TEKDB/urls.py index 536e2d27..eb694a67 100644 --- a/TEKDB/TEKDB/urls.py +++ b/TEKDB/TEKDB/urls.py @@ -77,7 +77,10 @@ views.ResourceActivityAutocompleteView.as_view(), name="select2_fk_resourceactivity", ), - re_path(r"^admin_async_upload/", include("admin_async_upload.urls")), + re_path( + r"^django_resumable_async_upload/", + include("django_resumable_async_upload.urls"), + ), path("", include("explore.urls")), ] urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index fc9e8b00..fac43bbe 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -3,14 +3,6 @@ # Exit on errors set -e -# Install editable package if the directory is mounted -if [ -d "/django-resumable-async-upload" ]; then - echo "Installing django-resumable-async-upload in editable mode for local development..." - pip install -e /django-resumable-async-upload -else - echo "django-resumable-async-upload directory not found, skipping editable install" -fi - # If a SQL_HOST is provided, wait for Postgres to become available before running # migrations. This prevents race conditions when using docker-compose where the # web container starts before the DB is ready. diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index b7afcd26..9a0cf228 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -15,6 +15,7 @@ psycopg2-binary psutil django-filebrowser-no-grappelli>=4.0.0,<5.0.0 XlsxWriter +django-resumable-async-upload #-e git+https://github.com/dominno/django-moderation.git@master#egg=moderation #DEPENDENCIES diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0db8d15d..f865e83a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,7 +40,6 @@ services: - "8000:8000" volumes: - ../TEKDB:/usr/src/app - - ../../django-resumable-async-upload:/django-resumable-async-upload volumes: tekdb_db_data: From 295dd49cfe6c54a60a29533c22e553512ebda6fc Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 10:56:22 -0800 Subject: [PATCH 14/98] use python 3.11 in gh action run-tests --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 83c10b7a..2b0aa017 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,7 +32,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.10' + python-version: '3.11' - name: Install system dependencies run: | From be3da99a297a4a415be9585ce1f4747534c8ba58 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 11:14:39 -0800 Subject: [PATCH 15/98] add migration for async file fields --- .../0026_alter_media_mediafile_and_more.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py diff --git a/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py new file mode 100644 index 00000000..76ca5d54 --- /dev/null +++ b/TEKDB/TEKDB/migrations/0026_alter_media_mediafile_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.27 on 2026-01-30 19:11 + +from django.db import migrations, models +import django_resumable_async_upload.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('TEKDB', '0025_alter_mediabulkupload_mediabulkname'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='mediafile', + field=django_resumable_async_upload.models.AsyncFileField(blank=True, db_column='mediafile', max_files=1, max_length=255, null=True, upload_to='', verbose_name='file'), + ), + migrations.AlterField( + model_name='mediabulkupload', + name='mediabulkname', + field=models.CharField(blank=True, default='Bulk Upload on 2026-01-30', max_length=255, null=True, verbose_name='name'), + ), + ] From 7404d034b8471fb68cb492dd79b6102a3637d9b6 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 30 Jan 2026 11:50:42 -0800 Subject: [PATCH 16/98] revert tests to run on python 3.10 --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 2b0aa017..83c10b7a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,7 +32,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.11' + python-version: '3.10' - name: Install system dependencies run: | From 27d0ff68a59ef4acdb27f8cdbe73d22e03f53835 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 14:29:13 -0800 Subject: [PATCH 17/98] use .env.prod in docker-compose.prod.yaml --- .gitignore | 1 + docker/docker-compose.prod.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c5ba5117..c77ac2b2 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ celerybeat-schedule # dotenv .env +.env.prod # virtualenv venv/ diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index 872e7c53..2598a35b 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -26,7 +26,7 @@ services: depends_on: - db env_file: - - .env.dev + - .env.prod environment: ALLOWED_HOSTS: ${ALLOWED_HOSTS} DEBUG: ${DEBUG} From 5e5cc5bbf30fe7f0c4a368415e66415db5f2f053 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:13:11 -0800 Subject: [PATCH 18/98] add github action to publish docker image to github packages --- .../create-and-publish-docker-images.yml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/create-and-publish-docker-images.yml diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml new file mode 100644 index 00000000..37256b53 --- /dev/null +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -0,0 +1,58 @@ +# source: https://docs.github.com/en/actions/tutorials/publish-packages/publish-docker-images#publishing-images-to-docker-hub-and-github-packages +name: Create and publish a Docker image + +# manually trigger while testing +on: + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + # + steps: + - name: Checkout repository + uses: actions/checkout@v5 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image + id: push + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v3 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + From e91eb06f4849892ec69b6c3b10c4cf4a75cba93d Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:20:27 -0800 Subject: [PATCH 19/98] point push step to prod dockerfile --- .github/workflows/create-and-publish-docker-images.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 37256b53..ffea8093 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -43,9 +43,10 @@ jobs: id: push uses: docker/build-push-action@v6 with: - context: . + context: ./TEKDB + file: ./TEKDB/prod.Dockerfile push: true - tags: ${{ steps.meta.outputs.tags }} + tags: ghcr.io/${{ github.repository }}/web:latest,ghcr.io/${{ github.repository }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). From ec83f25d21d5add9c63b62c94e691c17d65914d1 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:24:15 -0800 Subject: [PATCH 20/98] try triggering action on push --- .github/workflows/create-and-publish-docker-images.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index ffea8093..eb90ec3d 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -2,8 +2,7 @@ name: Create and publish a Docker image # manually trigger while testing -on: - workflow_dispatch: +on: [push, workflow_dispatch] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From 453f95e339610b8c2272e438ecd09dec6146adb0 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:35:37 -0800 Subject: [PATCH 21/98] try generating slug for repo name --- .github/workflows/create-and-publish-docker-images.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index eb90ec3d..85e9e4f1 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -35,9 +35,16 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Format repo slug + uses: actions/github-script@v4 + id: repo_slug + with: + result-encoding: string + script: return `ghcr.io/${github.repository.toLowerCase()}` # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Build and push Docker image id: push uses: docker/build-push-action@v6 @@ -45,8 +52,11 @@ jobs: context: ./TEKDB file: ./TEKDB/prod.Dockerfile push: true + builder: ${{ steps.buildx.outputs.name }} tags: ghcr.io/${{ github.repository }}/web:latest,ghcr.io/${{ github.repository }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}:main + cache-to: type=inline # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). - name: Generate artifact attestation From 4d93d8d5bdb1781c3a7d056c28b664880a81cf25 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:41:08 -0800 Subject: [PATCH 22/98] try getting slug for repo name as string --- .github/workflows/create-and-publish-docker-images.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 85e9e4f1..841295de 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -36,11 +36,12 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Format repo slug - uses: actions/github-script@v4 + uses: actions/github-script@v8 id: repo_slug with: result-encoding: string - script: return `ghcr.io/${github.repository.toLowerCase()}` + script: | + return `ghcr.io/${process.env.GITHUB_REPOSITORY.toLowerCase()}` # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. From 43916a5b11a81d29642eac52192ef7b4200cde92 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:43:54 -0800 Subject: [PATCH 23/98] try again to use lowercase repo name --- .github/workflows/create-and-publish-docker-images.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 841295de..3d212206 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -7,7 +7,6 @@ on: [push, workflow_dispatch] # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} jobs: build-and-push-image: @@ -35,6 +34,10 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # note: IMAGE_NAME will be set in the next step via GITHUB_ENV + - name: Set lowercase image name + run: | + echo "IMAGE_NAME=$(echo \"${GITHUB_REPOSITORY}\" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: Format repo slug uses: actions/github-script@v8 id: repo_slug @@ -53,8 +56,7 @@ jobs: context: ./TEKDB file: ./TEKDB/prod.Dockerfile push: true - builder: ${{ steps.buildx.outputs.name }} - tags: ghcr.io/${{ github.repository }}/web:latest,ghcr.io/${{ github.repository }}/web:${{ github.sha }} + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}:main cache-to: type=inline From 521e7509cb4f36d0c3cc93b23535c5fac0b85858 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:46:19 -0800 Subject: [PATCH 24/98] try setting lowercase IMAGE_NAME env var --- .github/workflows/create-and-publish-docker-images.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 3d212206..f95f4f63 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -29,15 +29,15 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Set lowercase image name + run: | + IMAGE_NAME=$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]') + echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # note: IMAGE_NAME will be set in the next step via GITHUB_ENV - - name: Set lowercase image name - run: | - echo "IMAGE_NAME=$(echo \"${GITHUB_REPOSITORY}\" | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV - name: Format repo slug uses: actions/github-script@v8 id: repo_slug From 05fcd1b18c342049421b519eddd7a97bc3c8aadf Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 15:49:41 -0800 Subject: [PATCH 25/98] add prod dockerfile --- TEKDB/prod.Dockerfile | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 TEKDB/prod.Dockerfile diff --git a/TEKDB/prod.Dockerfile b/TEKDB/prod.Dockerfile new file mode 100644 index 00000000..ca2d2e24 --- /dev/null +++ b/TEKDB/prod.Dockerfile @@ -0,0 +1,46 @@ +FROM python:3.11-slim + +# Prevent Python from writing .pyc files and enable unbuffered stdout/stderr +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV PIP_NO_CACHE_DIR=1 + +# Install system dependencies +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + postgresql-client \ + gcc \ + gdal-bin \ + libgdal-dev \ + libgeos-dev \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /usr/src/app + +# Copy requirements first (cache pip install step when dependencies don't change) +COPY requirements.txt requirements_linux.txt /usr/src/app/ + +# Upgrade pip and install Python dependencies +# Note: editable packages (-e) will be installed at runtime via entrypoint.sh +RUN pip install --upgrade pip \ + && pip install -r requirements.txt -r requirements_linux.txt + +# Copy the application code +COPY . /usr/src/app + +# Copy and make entrypoint executable. The repository contains `docker/entrypoint.sh` +# which runs collectstatic, migrations and launches uWSGI. +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000) +EXPOSE 8000 + +# Default settings module (can be overridden at runtime) +ENV DJANGO_SETTINGS_MODULE=TEKDB.settings + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +# use prod server in prod Dockerfile +CMD ["prod"] \ No newline at end of file From 76b4ab0e55886ea9830dda55d868c73ce166afe7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 16:12:03 -0800 Subject: [PATCH 26/98] remove cache lines for build and push step --- .github/workflows/create-and-publish-docker-images.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index f95f4f63..be42c4fb 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -58,8 +58,6 @@ jobs: push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=${{ steps.repo_slug.outputs.result }}:main - cache-to: type=inline # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). - name: Generate artifact attestation From 5fb47f561ed5c21bfa8b1853b41f54d515252955 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 16:25:26 -0800 Subject: [PATCH 27/98] try token change --- .github/workflows/create-and-publish-docker-images.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index be42c4fb..b3eca98e 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -11,13 +11,12 @@ env: jobs: build-and-push-image: runs-on: ubuntu-latest - # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + # Sets the permissions granted to the `PACKAGE_TOKEN` for the actions in this job. permissions: contents: read packages: write attestations: write id-token: write - # steps: - name: Checkout repository uses: actions/checkout@v5 @@ -27,7 +26,7 @@ jobs: with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.PACKAGE_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Set lowercase image name run: | From 942ea136f6329ee49252b4723683d82dc6b5f955 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Feb 2026 16:45:15 -0800 Subject: [PATCH 28/98] remove push from trigger options --- .github/workflows/create-and-publish-docker-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index b3eca98e..658a2459 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -2,7 +2,8 @@ name: Create and publish a Docker image # manually trigger while testing -on: [push, workflow_dispatch] +on: + workflow_dispatch # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From c83e88ce5e8dd1bf6cd3a44a1a4915998b3a862e Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Feb 2026 10:47:33 -0800 Subject: [PATCH 29/98] use common docker compose and use extends --- docker/{docker-compose.yml => common.yaml} | 7 ++-- docker/docker-compose.prod.yaml | 42 ++++------------------ docker/docker-compose.yaml | 13 +++++++ 3 files changed, 21 insertions(+), 41 deletions(-) rename docker/{docker-compose.yml => common.yaml} (94%) create mode 100644 docker/docker-compose.yaml diff --git a/docker/docker-compose.yml b/docker/common.yaml similarity index 94% rename from docker/docker-compose.yml rename to docker/common.yaml index f865e83a..3ee08bef 100644 --- a/docker/docker-compose.yml +++ b/docker/common.yaml @@ -16,7 +16,7 @@ services: interval: 10s timeout: 5s retries: 5 - + web: build: context: ../TEKDB/ @@ -39,7 +39,4 @@ services: ports: - "8000:8000" volumes: - - ../TEKDB:/usr/src/app - -volumes: - tekdb_db_data: + - ../TEKDB:/usr/src/app \ No newline at end of file diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index 2598a35b..c063379b 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -1,46 +1,16 @@ services: db: - image: postgis/postgis:15-3.4 - restart: always - platform: linux/amd64 - environment: - POSTGRES_DB: ${SQL_DATABASE} - POSTGRES_USER: ${SQL_USER} - POSTGRES_PASSWORD: ${SQL_PASSWORD} - volumes: - - tekdb_db_data:/var/lib/postgresql/data - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${SQL_USER} -d ${SQL_DATABASE} -h localhost -p ${SQL_PORT}"] - interval: 10s - timeout: 5s - retries: 5 + extends: + file: common.yaml + service: db web: - build: - context: ../TEKDB/ - dockerfile: ../TEKDB/Dockerfile + extends: + file: common.yaml + service: web command: ["prod"] - restart: unless-stopped - depends_on: - - db env_file: - .env.prod - environment: - ALLOWED_HOSTS: ${ALLOWED_HOSTS} - DEBUG: ${DEBUG} - SQL_ENGINE: ${SQL_ENGINE} - SQL_HOST: ${SQL_HOST} - SQL_PORT: ${SQL_PORT} - SQL_DATABASE: ${SQL_DATABASE} - SQL_USER: ${SQL_USER} - SQL_PASSWORD: ${SQL_PASSWORD} - SECRET_KEY: ${SECRET_KEY} - ports: - - "8000:8000" - volumes: - - ../TEKDB:/usr/src/app volumes: tekdb_db_data: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..5a99ff86 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,13 @@ +services: + db: + extends: + file: common.yaml + service: db + + web: + extends: + file: common.yaml + service: web + +volumes: + tekdb_db_data: From 3e60eaaeb4cc55553250cf5b25d93ce396c6d9a2 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Feb 2026 12:54:58 -0800 Subject: [PATCH 30/98] wip: add gh action to push to ECR and deploy to app runner --- .github/workflows/deploy-app-runner.yml | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/deploy-app-runner.yml diff --git a/.github/workflows/deploy-app-runner.yml b/.github/workflows/deploy-app-runner.yml new file mode 100644 index 00000000..c9d90965 --- /dev/null +++ b/.github/workflows/deploy-app-runner.yml @@ -0,0 +1,62 @@ +name: Publish to ECR and Deploy to App Runner +on: + push: + branches: [develop] # Trigger workflow on git push to develop branch + workflow_dispatch: # Allow manual invocation of the workflow + +jobs: + deploy: + runs-on: ubuntu-latest + # These permissions are needed to interact with GitHub's OIDC Token endpoint. + permissions: + id-token: write + contents: read + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + persist-credentials: false + + - name: Configure AWS credentials + id: aws-credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + # Use GitHub OIDC provider + role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + # steps to build, tag, and push Docker image to ECR + # tag with both latest and git sha + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: tekdb + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest + echo "::set-output name=image-latest::$ECR_REGISTRY/$ECR_REPOSITORY:latest" + + # - name: Deploy to App Runner Image + # id: deploy-apprunner + # uses: awslabs/amazon-app-runner-deploy@main + # with: + # service: app-runner-git-deploy-service + # image: ${{ steps.build-image.outputs.image }} + # access-role-arn: ${{ secrets.ROLE_ARN }} + # region: ${{ secrets.AWS_REGION }} + # cpu : 1 + # memory : 2 + # wait-for-service-stability-seconds: 1200 + + # - name: App Runner URL + # run: echo "App runner URL ${{ steps.deploy-apprunner.outputs.service-url }}" \ No newline at end of file From cddcf48dd50a715302a9051f3db2f30791c2ea93 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 23 Feb 2026 16:04:33 -0800 Subject: [PATCH 31/98] set 5432 as default db port --- TEKDB/TEKDB/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 49af92a8..21028c6b 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -127,7 +127,7 @@ "USER": os.environ.get("SQL_USER", "postgres"), "PASSWORD": os.environ.get("SQL_PASSWORD", None), "HOST": os.environ.get("SQL_HOST", "db"), - "PORT": os.environ.get("SQL_PORT", None), + "PORT": os.environ.get("SQL_PORT", 5432), } } From bd97aacbd2f5387f3109aefaa7621469325f9823 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 23 Feb 2026 16:56:56 -0800 Subject: [PATCH 32/98] use nginx proxy in prod setup --- TEKDB/entrypoint.sh | 4 ++-- docker/docker-compose.prod.yaml | 18 ++++++++++++++++++ proxy/Dockerfile | 9 +++++---- proxy/default.conf | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index fac43bbe..182563da 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -32,8 +32,8 @@ if [ "$(python manage.py shell -c 'from TEKDB.models import LookupPlanningUnit; fi if [ "$1" = "prod" ]; then - echo "Starting uWSGI (HTTP) on :8000" - uwsgi --http :8000 --master --enable-threads --module TEKDB.wsgi + echo "Starting uWSGI (socket) on :8000" + uwsgi --socket :8000 --master --enable-threads --module TEKDB.wsgi elif [ "$1" = "dev" ]; then echo "Starting python development server on :8000" python manage.py runserver 0.0.0.0:8000 diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index c063379b..2f0b55b9 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -11,6 +11,24 @@ services: command: ["prod"] env_file: - .env.prod + ports: [] + volumes: + - static_volume:/usr/src/app/static + - media_volume:/usr/src/app/media + proxy: + build: + context: ../proxy/ + dockerfile: ../proxy/Dockerfile + restart: unless-stopped + depends_on: + - web + ports: + - "8080:8080" + volumes: + - static_volume:/vol/static/static:ro + - media_volume:/vol/static/media:ro volumes: tekdb_db_data: + static_volume: + media_volume: diff --git a/proxy/Dockerfile b/proxy/Dockerfile index b7fd7b32..978935c0 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,11 +1,12 @@ FROM nginxinc/nginx-unprivileged:1-alpine +USER root + COPY ./default.conf /etc/nginx/conf.d/default.conf COPY ./uwsgi_params /etc/nginx/uwsgi_params -USER root - -RUN mkdir -p /vol/static -RUN chmod 755 /vol/static +RUN chmod 644 /etc/nginx/conf.d/default.conf /etc/nginx/uwsgi_params && \ + mkdir -p /vol/static && \ + chmod 755 /vol/static USER nginx diff --git a/proxy/default.conf b/proxy/default.conf index 99e1e714..8ef2d0a5 100644 --- a/proxy/default.conf +++ b/proxy/default.conf @@ -8,7 +8,7 @@ server { alias /vol/static/media; } location / { - uwsgi_pass app:8000; + uwsgi_pass web:8000; include /etc/nginx/uwsgi_params; } } From fd1ec8987df1502ada2392a25d5f671a389f1281 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 24 Feb 2026 09:18:03 -0800 Subject: [PATCH 33/98] add docker compose for testing prod set up locally --- docker/docker-compose.prod.local.yaml | 34 +++++++++++++++++++++++++++ docker/docker-compose.prod.yaml | 7 +++--- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 docker/docker-compose.prod.local.yaml diff --git a/docker/docker-compose.prod.local.yaml b/docker/docker-compose.prod.local.yaml new file mode 100644 index 00000000..2f0b55b9 --- /dev/null +++ b/docker/docker-compose.prod.local.yaml @@ -0,0 +1,34 @@ +services: + db: + extends: + file: common.yaml + service: db + + web: + extends: + file: common.yaml + service: web + command: ["prod"] + env_file: + - .env.prod + ports: [] + volumes: + - static_volume:/usr/src/app/static + - media_volume:/usr/src/app/media + proxy: + build: + context: ../proxy/ + dockerfile: ../proxy/Dockerfile + restart: unless-stopped + depends_on: + - web + ports: + - "8080:8080" + volumes: + - static_volume:/vol/static/static:ro + - media_volume:/vol/static/media:ro + +volumes: + tekdb_db_data: + static_volume: + media_volume: diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index 2f0b55b9..2105577a 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -8,6 +8,7 @@ services: extends: file: common.yaml service: web + image: ${ITKDB_ECR_PATH}:latest command: ["prod"] env_file: - .env.prod @@ -16,14 +17,12 @@ services: - static_volume:/usr/src/app/static - media_volume:/usr/src/app/media proxy: - build: - context: ../proxy/ - dockerfile: ../proxy/Dockerfile + image: ${ITKDB_PROXY_ECR_PATH}:latest restart: unless-stopped depends_on: - web ports: - - "8080:8080" + - "80:8080" volumes: - static_volume:/vol/static/static:ro - media_volume:/vol/static/media:ro From 7f91476b5501f2678db41cb9cbba108d19b3a43e Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 24 Feb 2026 11:21:47 -0800 Subject: [PATCH 34/98] add OpenTofu/terraform to gitignore --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c77ac2b2..a3f79ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -99,4 +99,10 @@ ENV/ .ropeproject #vscode settings -.vscode/ \ No newline at end of file +.vscode/ + +# OpenTofu/Terraform +.terraform/ +terraform.tfstate +terraform.tfstate.backup +*.tfvars \ No newline at end of file From 60d0dcc0277c7bc711a1371ff036a0ee2c7c84f3 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 25 Feb 2026 16:41:56 -0800 Subject: [PATCH 35/98] add tf/opentofu IaC --- docker/common.yaml | 14 +--------- docker/docker-compose.yaml | 12 +++++++++ infra/ec2.tf | 55 ++++++++++++++++++++++++++++++++++++++ infra/ecr.tf | 17 ++++++++++++ infra/iam.tf | 28 +++++++++++++++++++ infra/main.tf | 52 +++++++++++++++++++++++++++++++++++ infra/networking.tf | 53 ++++++++++++++++++++++++++++++++++++ infra/outputs.tf | 16 +++++++++++ infra/terraform.tf | 17 ++++++++++++ 9 files changed, 251 insertions(+), 13 deletions(-) create mode 100644 infra/ec2.tf create mode 100644 infra/ecr.tf create mode 100644 infra/iam.tf create mode 100644 infra/main.tf create mode 100644 infra/networking.tf create mode 100644 infra/outputs.tf create mode 100644 infra/terraform.tf diff --git a/docker/common.yaml b/docker/common.yaml index 3ee08bef..2d58c8e7 100644 --- a/docker/common.yaml +++ b/docker/common.yaml @@ -26,17 +26,5 @@ services: - db env_file: - .env.dev - environment: - ALLOWED_HOSTS: ${ALLOWED_HOSTS} - DEBUG: ${DEBUG} - SQL_ENGINE: ${SQL_ENGINE} - SQL_HOST: ${SQL_HOST} - SQL_PORT: ${SQL_PORT} - SQL_DATABASE: ${SQL_DATABASE} - SQL_USER: ${SQL_USER} - SQL_PASSWORD: ${SQL_PASSWORD} - SECRET_KEY: ${SECRET_KEY} ports: - - "8000:8000" - volumes: - - ../TEKDB:/usr/src/app \ No newline at end of file + - "8000:8000" \ No newline at end of file diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5a99ff86..5969649f 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -8,6 +8,18 @@ services: extends: file: common.yaml service: web + volumes: + - ../TEKDB:/usr/src/app + environment: + ALLOWED_HOSTS: ${ALLOWED_HOSTS} + DEBUG: ${DEBUG} + SQL_ENGINE: ${SQL_ENGINE} + SQL_HOST: ${SQL_HOST} + SQL_PORT: ${SQL_PORT} + SQL_DATABASE: ${SQL_DATABASE} + SQL_USER: ${SQL_USER} + SQL_PASSWORD: ${SQL_PASSWORD} + SECRET_KEY: ${SECRET_KEY} volumes: tekdb_db_data: diff --git a/infra/ec2.tf b/infra/ec2.tf new file mode 100644 index 00000000..5242a645 --- /dev/null +++ b/infra/ec2.tf @@ -0,0 +1,55 @@ +resource "aws_key_pair" "itkdb" { + key_name = "${var.project_name}-key" + public_key = var.ssh_public_key + + tags = { + Project = "${var.project_name}-staging" + } +} + +resource "aws_instance" "itkdb" { + ami = var.ec2_ami + instance_type = var.ec2_instance_type + key_name = aws_key_pair.itkdb.key_name + vpc_security_group_ids = [aws_security_group.itkdb.id] + iam_instance_profile = aws_iam_instance_profile.ec2_profile.name + subnet_id = tolist(data.aws_subnets.default.ids)[0] + user_data_replace_on_change = true + + # TODO: fix this! currently does not work + # Install Docker and AWS CLI v2 on first boot + user_data = <<-EOF + #!/bin/bash + set -e + sudo apt update + sudo apt install -y docker + systemctl start docker + systemctl enable docker + usermod -aG docker ubuntu + sudo snap install aws-cli --classic + unzip awscliv2.zip + sudo ./aws/install + EOF + + root_block_device { + volume_size = 20 + volume_type = "gp3" + encrypted = true + } + + tags = { + Name = "${var.project_name}-staging-server" + Project = var.project_name + } +} + +# Elastic IP so the address never changes across stop/start +resource "aws_eip" "itkdb" { + instance = aws_instance.itkdb.id + domain = "vpc" + + tags = { + Name = "${var.project_name}-staging-eip" + Project = var.project_name + } +} \ No newline at end of file diff --git a/infra/ecr.tf b/infra/ecr.tf new file mode 100644 index 00000000..9a5c9826 --- /dev/null +++ b/infra/ecr.tf @@ -0,0 +1,17 @@ +resource "aws_ecr_repository" "web" { + name = "ecotrust/${var.project_name}" + image_tag_mutability = "MUTABLE" + + tags = { + Project = var.project_name + } +} + +resource "aws_ecr_repository" "proxy" { + name = "ecotrust/${var.project_name}-proxy" + image_tag_mutability = "MUTABLE" + + tags = { + Project = var.project_name + } +} diff --git a/infra/iam.tf b/infra/iam.tf new file mode 100644 index 00000000..fcf46649 --- /dev/null +++ b/infra/iam.tf @@ -0,0 +1,28 @@ +# IAM role that allows the EC2 instance to pull images from ECR +resource "aws_iam_role" "ec2_ecr_role" { + name = "${var.project_name}-ec2-ecr-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + }] + }) + + tags = { + Project = var.project_name + } +} + +resource "aws_iam_role_policy_attachment" "ecr_read" { + role = aws_iam_role.ec2_ecr_role.name + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" +} + +# Instance profile wraps the role so EC2 can use it +resource "aws_iam_instance_profile" "ec2_profile" { + name = "${var.project_name}-ec2-profile" + role = aws_iam_role.ec2_ecr_role.name +} \ No newline at end of file diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 00000000..57ebd1bd --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,52 @@ +provider "aws" { + region = var.aws_region +} + +variable "aws_region" { + description = "AWS region to deploy into" + type = string + default = "us-west-2" +} + +variable "project_name" { + description = "Project name used to prefix all resources" + type = string + default = "itkdb" +} + +variable "aws_profile" { + description = "AWS profile to use" + type = string + default = "default" +} + +variable "bucket_name" { + description = "S3 bucket name for Terraform state (must be globally unique)" + type = string + default = "itkdb-tf-state" +} + +variable "ec2_instance_type" { + description = "EC2 instance type" + type = string + default = "t3.small" +} + +variable "ec2_ami" { + description = "Ubuntu 24.04 LTS AMI ID (region-specific — update if changing region)" + type = string + # Ubuntu 24.04 LTS us-west-1 — check https://cloud-images.ubuntu.com/locator/ec2/ for your region + default = "ami-06b527a1e4cb6f265" +} + +variable "ssh_public_key" { + description = "SSH public key to install on the EC2 instance (contents of your .pub file)" + type = string + sensitive = true +} + +variable "allowed_ssh_cidr" { + description = "CIDR block allowed to SSH into the EC2 instance (use your IP: x.x.x.x/32)" + type = string + default = "0.0.0.0/0" # Restrict this to your IP in production! +} \ No newline at end of file diff --git a/infra/networking.tf b/infra/networking.tf new file mode 100644 index 00000000..fee43725 --- /dev/null +++ b/infra/networking.tf @@ -0,0 +1,53 @@ +data "aws_vpc" "default" { + default = true +} + +data "aws_subnets" "default" { + filter { + name = "vpc-id" + values = [data.aws_vpc.default.id] + } +} + +resource "aws_security_group" "itkdb" { + name = "${var.project_name}-sg" + description = "Security group for ITKDB Staging EC2 instance" + vpc_id = data.aws_vpc.default.id + + # HTTP + ingress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # HTTPS (for when you add SSL via certbot/nginx later) + ingress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + # SSH — restrict to your IP in production! + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [var.allowed_ssh_cidr] + } + + # Allow all outbound (needed for ECR pulls, apt, etc.) + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-sg" + Project = var.project_name + } +} \ No newline at end of file diff --git a/infra/outputs.tf b/infra/outputs.tf new file mode 100644 index 00000000..960acf91 --- /dev/null +++ b/infra/outputs.tf @@ -0,0 +1,16 @@ +output "ec2_public_ip" { + description = "Elastic IP of the EC2 instance — use this for DNS and GitHub secrets" + value = aws_eip.itkdb.public_ip +} + +output "ecr_web_url" { + description = "ECR URL for the web image" + value = aws_ecr_repository.web.repository_url +} + +output "ecr_proxy_url" { + description = "ECR URL for the proxy image" + value = aws_ecr_repository.proxy.repository_url +} + +data "aws_caller_identity" "current" {} \ No newline at end of file diff --git a/infra/terraform.tf b/infra/terraform.tf new file mode 100644 index 00000000..11c04bca --- /dev/null +++ b/infra/terraform.tf @@ -0,0 +1,17 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.92" + } + } + + required_version = ">= 1.2" + + backend "s3" { + bucket = "itkdb-tf-state" + key = "staging/terraform.tfstate" + region = var.aws_region + profile = "default" + } +} \ No newline at end of file From e8e2c00dcd41cf17ac7f9ddea4624a59e040670f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 26 Feb 2026 10:33:04 -0800 Subject: [PATCH 36/98] track terraform.lock.hcl --- infra/.terraform.lock.hcl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 infra/.terraform.lock.hcl diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 00000000..900ce952 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,20 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/aws" { + version = "5.100.0" + constraints = "~> 5.92" + hashes = [ + "h1:BrNG7eFOdRrRRbHdvrTjMJ8X8Oh/tiegURiKf7J2db8=", + "zh:1a41f3ee26720fee7a9a0a361890632a1701b5dc1cf5355dc651ddbe115682ff", + "zh:30457f36690c19307921885cc5e72b9dbeba369445815903acd5c39ac0e41e7a", + "zh:42c22674d5f23f6309eaf3ac3a4f1f8b66b566c1efe1dcb0dd2fb30c17ce1f78", + "zh:4cc271c795ff8ce6479ec2d11a8ba65a0a9ed6331def6693f4b9dccb6e662838", + "zh:60932aa376bb8c87cd1971240063d9d38ba6a55502c867fdbb9f5361dc93d003", + "zh:864e42784bde77b18393ebfcc0104cea9123da5f4392e8a059789e296952eefa", + "zh:9750423138bb01ecaa5cec1a6691664f7783d301fb1628d3b64a231b6b564e0e", + "zh:e5d30c4dec271ef9d6fe09f48237ec6cfea1036848f835b4e47f274b48bda5a7", + "zh:e62bd314ae97b43d782e0841b13e68a3f8ec85cc762004f973ce5ce7b6cdbfd0", + "zh:ea851a3c072528a4445ac6236ba2ce58ffc99ec466019b0bd0e4adde63a248e4", + ] +} From 1d46d36a20c83b5b20484a643f7b78603ba8e80f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 26 Feb 2026 12:01:07 -0800 Subject: [PATCH 37/98] remove deploy app runner gh action workflow --- .github/workflows/deploy-app-runner.yml | 62 ------------------------- 1 file changed, 62 deletions(-) delete mode 100644 .github/workflows/deploy-app-runner.yml diff --git a/.github/workflows/deploy-app-runner.yml b/.github/workflows/deploy-app-runner.yml deleted file mode 100644 index c9d90965..00000000 --- a/.github/workflows/deploy-app-runner.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: Publish to ECR and Deploy to App Runner -on: - push: - branches: [develop] # Trigger workflow on git push to develop branch - workflow_dispatch: # Allow manual invocation of the workflow - -jobs: - deploy: - runs-on: ubuntu-latest - # These permissions are needed to interact with GitHub's OIDC Token endpoint. - permissions: - id-token: write - contents: read - - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - persist-credentials: false - - - name: Configure AWS credentials - id: aws-credentials - uses: aws-actions/configure-aws-credentials@v5 - with: - # Use GitHub OIDC provider - role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v1 - - # steps to build, tag, and push Docker image to ECR - # tag with both latest and git sha - - name: Build, tag, and push image to Amazon ECR - id: build-image - env: - ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - ECR_REPOSITORY: tekdb - IMAGE_TAG: ${{ github.sha }} - run: | - docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . - docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG - echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" - docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest - docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest - echo "::set-output name=image-latest::$ECR_REGISTRY/$ECR_REPOSITORY:latest" - - # - name: Deploy to App Runner Image - # id: deploy-apprunner - # uses: awslabs/amazon-app-runner-deploy@main - # with: - # service: app-runner-git-deploy-service - # image: ${{ steps.build-image.outputs.image }} - # access-role-arn: ${{ secrets.ROLE_ARN }} - # region: ${{ secrets.AWS_REGION }} - # cpu : 1 - # memory : 2 - # wait-for-service-stability-seconds: 1200 - - # - name: App Runner URL - # run: echo "App runner URL ${{ steps.deploy-apprunner.outputs.service-url }}" \ No newline at end of file From 476f04bf874ce67ae84056bd5c05429813a920b1 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 26 Feb 2026 14:50:51 -0800 Subject: [PATCH 38/98] use commands in docker compose not dockerfiles; remove prod.Dockerfile --- .../create-and-publish-docker-images.yml | 2 +- TEKDB/Dockerfile | 2 - TEKDB/entrypoint.sh | 3 ++ TEKDB/prod.Dockerfile | 46 ------------------- docker/docker-compose.prod.local.yaml | 4 +- docker/docker-compose.yaml | 1 + 6 files changed, 7 insertions(+), 51 deletions(-) delete mode 100644 TEKDB/prod.Dockerfile diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 658a2459..d58cffc7 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -54,7 +54,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./TEKDB - file: ./TEKDB/prod.Dockerfile + file: ./TEKDB/Dockerfile push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index 1ba7ac1f..78fb2bd3 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -42,5 +42,3 @@ ENV DJANGO_SETTINGS_MODULE=TEKDB.settings ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] -# use development server by default -CMD ["dev"] diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index 182563da..a0644cd0 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -34,6 +34,9 @@ fi if [ "$1" = "prod" ]; then echo "Starting uWSGI (socket) on :8000" uwsgi --socket :8000 --master --enable-threads --module TEKDB.wsgi +elif [ "$1" = "prod-local" ]; then + echo "Starting uWSGI (http) on :8000 with local settings" + uwsgi --http :8000 --master --enable-threads --module TEKDB.wsgi elif [ "$1" = "dev" ]; then echo "Starting python development server on :8000" python manage.py runserver 0.0.0.0:8000 diff --git a/TEKDB/prod.Dockerfile b/TEKDB/prod.Dockerfile deleted file mode 100644 index ca2d2e24..00000000 --- a/TEKDB/prod.Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM python:3.11-slim - -# Prevent Python from writing .pyc files and enable unbuffered stdout/stderr -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PIP_NO_CACHE_DIR=1 - -# Install system dependencies -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - postgresql-client \ - gcc \ - gdal-bin \ - libgdal-dev \ - libgeos-dev \ - && rm -rf /var/lib/apt/lists/* - -# Set working directory -WORKDIR /usr/src/app - -# Copy requirements first (cache pip install step when dependencies don't change) -COPY requirements.txt requirements_linux.txt /usr/src/app/ - -# Upgrade pip and install Python dependencies -# Note: editable packages (-e) will be installed at runtime via entrypoint.sh -RUN pip install --upgrade pip \ - && pip install -r requirements.txt -r requirements_linux.txt - -# Copy the application code -COPY . /usr/src/app - -# Copy and make entrypoint executable. The repository contains `docker/entrypoint.sh` -# which runs collectstatic, migrations and launches uWSGI. -COPY entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh - -# Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000) -EXPOSE 8000 - -# Default settings module (can be overridden at runtime) -ENV DJANGO_SETTINGS_MODULE=TEKDB.settings - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] - -# use prod server in prod Dockerfile -CMD ["prod"] \ No newline at end of file diff --git a/docker/docker-compose.prod.local.yaml b/docker/docker-compose.prod.local.yaml index 2f0b55b9..421e3d93 100644 --- a/docker/docker-compose.prod.local.yaml +++ b/docker/docker-compose.prod.local.yaml @@ -8,9 +8,9 @@ services: extends: file: common.yaml service: web - command: ["prod"] + command: ["prod-local"] env_file: - - .env.prod + - .env.dev ports: [] volumes: - static_volume:/usr/src/app/static diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5969649f..742ef053 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -8,6 +8,7 @@ services: extends: file: common.yaml service: web + command: ["dev"] volumes: - ../TEKDB:/usr/src/app environment: From 1d13c2db5248568799ec46be1ecdcfb8c25b1e12 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 27 Feb 2026 11:32:36 -0800 Subject: [PATCH 39/98] add user_data script to use on install --- infra/ec2.tf | 15 ++------------- infra/user_data.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) create mode 100644 infra/user_data.sh diff --git a/infra/ec2.tf b/infra/ec2.tf index 5242a645..8dcedb89 100644 --- a/infra/ec2.tf +++ b/infra/ec2.tf @@ -17,19 +17,8 @@ resource "aws_instance" "itkdb" { user_data_replace_on_change = true # TODO: fix this! currently does not work - # Install Docker and AWS CLI v2 on first boot - user_data = <<-EOF - #!/bin/bash - set -e - sudo apt update - sudo apt install -y docker - systemctl start docker - systemctl enable docker - usermod -aG docker ubuntu - sudo snap install aws-cli --classic - unzip awscliv2.zip - sudo ./aws/install - EOF + # Install Docker, AWS CLI v2, and git on first boot + user_data = file("user_data.sh") root_block_device { volume_size = 20 diff --git a/infra/user_data.sh b/infra/user_data.sh new file mode 100644 index 00000000..98c1b4b8 --- /dev/null +++ b/infra/user_data.sh @@ -0,0 +1,38 @@ +#!/bin/bash +sudo snap install aws-cli --classic # install aws cli to pull images from ECR +sudo apt-get update && sudo apt-get upgrade -y + +# https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository +sudo apt update -y +sudo apt install ca-certificates curl -y +sudo install -m 0755 -d /etc/apt/keyrings + +echo "Installing Docker's official GPG key..." +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + +sudo chmod a+r /etc/apt/keyrings/docker.asc + +echo "Adding Docker's official repository..." +sudo tee /etc/apt/sources.list.d/docker.sources < Date: Fri, 27 Feb 2026 11:35:50 -0800 Subject: [PATCH 40/98] remove TODO --- infra/ec2.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/infra/ec2.tf b/infra/ec2.tf index 8dcedb89..2ba60d5c 100644 --- a/infra/ec2.tf +++ b/infra/ec2.tf @@ -16,7 +16,6 @@ resource "aws_instance" "itkdb" { subnet_id = tolist(data.aws_subnets.default.ids)[0] user_data_replace_on_change = true - # TODO: fix this! currently does not work # Install Docker, AWS CLI v2, and git on first boot user_data = file("user_data.sh") From e5f6998a972528037af635e651a397144c07a9a1 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 09:41:29 -0800 Subject: [PATCH 41/98] clean up infra files --- infra/main.tf | 9 +-------- infra/outputs.tf | 4 +--- infra/terraform.tf | 2 +- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/infra/main.tf b/infra/main.tf index 57ebd1bd..7c1343e6 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -14,12 +14,6 @@ variable "project_name" { default = "itkdb" } -variable "aws_profile" { - description = "AWS profile to use" - type = string - default = "default" -} - variable "bucket_name" { description = "S3 bucket name for Terraform state (must be globally unique)" type = string @@ -35,7 +29,7 @@ variable "ec2_instance_type" { variable "ec2_ami" { description = "Ubuntu 24.04 LTS AMI ID (region-specific — update if changing region)" type = string - # Ubuntu 24.04 LTS us-west-1 — check https://cloud-images.ubuntu.com/locator/ec2/ for your region + # Ubuntu 24.04 LTS us-west-2 — check https://cloud-images.ubuntu.com/locator/ec2/ for your region default = "ami-06b527a1e4cb6f265" } @@ -48,5 +42,4 @@ variable "ssh_public_key" { variable "allowed_ssh_cidr" { description = "CIDR block allowed to SSH into the EC2 instance (use your IP: x.x.x.x/32)" type = string - default = "0.0.0.0/0" # Restrict this to your IP in production! } \ No newline at end of file diff --git a/infra/outputs.tf b/infra/outputs.tf index 960acf91..289e186c 100644 --- a/infra/outputs.tf +++ b/infra/outputs.tf @@ -11,6 +11,4 @@ output "ecr_web_url" { output "ecr_proxy_url" { description = "ECR URL for the proxy image" value = aws_ecr_repository.proxy.repository_url -} - -data "aws_caller_identity" "current" {} \ No newline at end of file +} \ No newline at end of file diff --git a/infra/terraform.tf b/infra/terraform.tf index 11c04bca..8a2b5a60 100644 --- a/infra/terraform.tf +++ b/infra/terraform.tf @@ -11,7 +11,7 @@ terraform { backend "s3" { bucket = "itkdb-tf-state" key = "staging/terraform.tfstate" - region = var.aws_region + region = "us-west-2" profile = "default" } } \ No newline at end of file From d55fbfe8b6db1996ee2d25e6aa09c0df7c05395c Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 09:49:47 -0800 Subject: [PATCH 42/98] add step in gh action to build and push proxy docker image --- .../workflows/create-and-publish-docker-images.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index d58cffc7..2d460ecf 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -49,7 +49,7 @@ jobs: # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. - - name: Build and push Docker image + - name: Build and push web Docker image id: push uses: docker/build-push-action@v6 with: @@ -58,6 +58,15 @@ jobs: push: true tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:${{ github.sha }} labels: ${{ steps.meta.outputs.labels }} + - name: Build and push proxy Docker image + id: push-proxy + uses: docker/build-push-action@v6 + with: + context: ./proxy + file: ./proxy/Dockerfile + push: true + tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy:${{ github.sha }} + labels: ${{ steps.meta.outputs.labels }} # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). - name: Generate artifact attestation From d26b0905ba55460555752c21c6f8d153756e9208 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 09:59:52 -0800 Subject: [PATCH 43/98] add action run on push for testing --- .github/workflows/create-and-publish-docker-images.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 2d460ecf..4b8236e0 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -3,7 +3,9 @@ name: Create and publish a Docker image # manually trigger while testing on: - workflow_dispatch + workflow_dispatch: + push: + # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From dda7f30ec0c2ee5b8466b58b05ad36d9d176864f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 10:18:59 -0800 Subject: [PATCH 44/98] remove format repo slug step in gh action --- .github/workflows/create-and-publish-docker-images.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 4b8236e0..57bfa186 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -40,13 +40,6 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Format repo slug - uses: actions/github-script@v8 - id: repo_slug - with: - result-encoding: string - script: | - return `ghcr.io/${process.env.GITHUB_REPOSITORY.toLowerCase()}` # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. From cedfa4163bed324dea726655c3712107a3c9056c Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 16:12:24 -0800 Subject: [PATCH 45/98] spin up docker containers in user_data script --- infra/ec2.tf | 13 ++++++- infra/main.tf | 49 ++++++++++++++++++++++++ infra/user_data.sh | 38 ------------------- infra/user_data.tftpl | 86 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 39 deletions(-) delete mode 100644 infra/user_data.sh create mode 100644 infra/user_data.tftpl diff --git a/infra/ec2.tf b/infra/ec2.tf index 2ba60d5c..70e066b9 100644 --- a/infra/ec2.tf +++ b/infra/ec2.tf @@ -17,7 +17,18 @@ resource "aws_instance" "itkdb" { user_data_replace_on_change = true # Install Docker, AWS CLI v2, and git on first boot - user_data = file("user_data.sh") + user_data = templatefile("user_data.tftpl", { + aws_region = var.aws_region + web_ecr_image_uri = var.web_ecr_image_uri + proxy_ecr_image_uri = var.proxy_ecr_image_uri + django_secret_key = var.django_secret_key + sql_host = var.sql_host + sql_db_name = var.sql_db_name + sql_db_user = var.sql_db_user + sql_db_password = var.sql_db_password + sql_port = var.sql_port + django_allowed_hosts = var.django_allowed_hosts + }) root_block_device { volume_size = 20 diff --git a/infra/main.tf b/infra/main.tf index 7c1343e6..e992b052 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -42,4 +42,53 @@ variable "ssh_public_key" { variable "allowed_ssh_cidr" { description = "CIDR block allowed to SSH into the EC2 instance (use your IP: x.x.x.x/32)" type = string +} + +# vars for Django environment variables +variable "django_secret_key" { + description = "Secret key for Django application" + type = string + sensitive = true +} + +variable "django_allowed_hosts" { + description = "Comma-separated list of allowed hosts for Django" + type = string +} + +variable "sql_db_name" { + description = "Name of the PostgreSQL database" + type = string +} + +variable "sql_db_user" { + description = "PostgreSQL database user" + type = string +} + +variable "sql_db_password" { + description = "PostgreSQL database password" + type = string + sensitive = true +} + +variable "sql_host" { + description = "Hostname for the PostgreSQL database" + type = string +} + +variable "sql_port" { + description = "Port for the PostgreSQL database" + type = number + default = 5432 +} + +variable "web_ecr_image_uri" { + description = "ECR image URI for the web application" + type = string +} + +variable "proxy_ecr_image_uri" { + description = "ECR image URI for the proxy" + type = string } \ No newline at end of file diff --git a/infra/user_data.sh b/infra/user_data.sh deleted file mode 100644 index 98c1b4b8..00000000 --- a/infra/user_data.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -sudo snap install aws-cli --classic # install aws cli to pull images from ECR -sudo apt-get update && sudo apt-get upgrade -y - -# https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository -sudo apt update -y -sudo apt install ca-certificates curl -y -sudo install -m 0755 -d /etc/apt/keyrings - -echo "Installing Docker's official GPG key..." -sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc - -sudo chmod a+r /etc/apt/keyrings/docker.asc - -echo "Adding Docker's official repository..." -sudo tee /etc/apt/sources.list.d/docker.sources < /TEKDB/docker/.env.prod < Date: Mon, 2 Mar 2026 16:22:56 -0800 Subject: [PATCH 46/98] fix syntax in attestation step --- .github/workflows/create-and-publish-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 57bfa186..cd02f323 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -67,7 +67,7 @@ jobs: - name: Generate artifact attestation uses: actions/attest-build-provenance@v3 with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true From ef93a8e19ec6b6cc8d12ca86f399c4f516577798 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 2 Mar 2026 16:29:49 -0800 Subject: [PATCH 47/98] fix attestation step for images --- .github/workflows/create-and-publish-docker-images.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index cd02f323..e636bd9f 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -64,10 +64,16 @@ jobs: labels: ${{ steps.meta.outputs.labels }} # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds). - - name: Generate artifact attestation + - name: Generate artifact attestation for web image uses: actions/attest-build-provenance@v3 with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true + - name: Generate artifact attestation for proxy image + uses: actions/attest-build-provenance@v3 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy + subject-digest: ${{ steps.push-proxy.outputs.digest }} + push-to-registry: true From 7cca80b07cc28c229cc67c429e5ed15ce03eb653 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 10:44:13 -0800 Subject: [PATCH 48/98] wip: gh action to build and push to ECR --- .github/workflows/deploy-ec2-staging.yml | 47 ++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/deploy-ec2-staging.yml diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml new file mode 100644 index 00000000..cd14a7e2 --- /dev/null +++ b/.github/workflows/deploy-ec2-staging.yml @@ -0,0 +1,47 @@ +name: Build, Push, and Deploy to EC2 Staging + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + build-push-and-deploy: + name: Build, Push to ECR, Deploy to EC2 Staging + runs-on: ubuntu-latest + permissions: + id-token: write + + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION}} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build, tag, and push web Docker image to ECR + id: build-web-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} + run: | + docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . + docker push $IMAGE_URI:$IMAGE_TAG + docker push $IMAGE_URI:latest + + - name: Build, tag, and push proxy Docker image to ECR + id: build-proxy-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} + run: | + docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . + docker push $IMAGE_URI:$IMAGE_TAG + docker push $IMAGE_URI:latest \ No newline at end of file From 763cc131fbbfd2cb9e50f42a97756fa8dbfe0100 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 10:46:35 -0800 Subject: [PATCH 49/98] update cadence for gh actions --- .github/workflows/create-and-publish-docker-images.yml | 1 - .github/workflows/deploy-ec2-staging.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index e636bd9f..3a0bf021 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -3,7 +3,6 @@ name: Create and publish a Docker image # manually trigger while testing on: - workflow_dispatch: push: diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index cd14a7e2..256fe9f5 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -1,6 +1,7 @@ name: Build, Push, and Deploy to EC2 Staging on: + push: release: types: [published] workflow_dispatch: From 728a1b8349f45d0a90597a78bdaab1206ebd004a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 11:01:44 -0800 Subject: [PATCH 50/98] wip: debug gh action --- .github/workflows/deploy-ec2-staging.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 256fe9f5..eff70fc7 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v5 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6 + uses: aws-actions/configure-aws-credentials@v6.0.0 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION}} @@ -36,6 +36,7 @@ jobs: docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . docker push $IMAGE_URI:$IMAGE_TAG docker push $IMAGE_URI:latest + echo "pushed web image" - name: Build, tag, and push proxy Docker image to ECR id: build-proxy-image @@ -45,4 +46,5 @@ jobs: run: | docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . docker push $IMAGE_URI:$IMAGE_TAG - docker push $IMAGE_URI:latest \ No newline at end of file + docker push $IMAGE_URI:latest + echo "pushed proxy image" \ No newline at end of file From 4b16ff5ff7125dd415584c66de6405e7c6ee721d Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 11:07:32 -0800 Subject: [PATCH 51/98] point to correct dockerfile locations --- .github/workflows/deploy-ec2-staging.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index eff70fc7..0a40ecb1 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v5 - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v6.0.0 + uses: aws-actions/configure-aws-credentials@v5 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION}} @@ -33,7 +33,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} run: | - docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . + docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest -f ./TEKDB/Dockerfile ./TEKDB docker push $IMAGE_URI:$IMAGE_TAG docker push $IMAGE_URI:latest echo "pushed web image" @@ -44,7 +44,7 @@ jobs: IMAGE_TAG: ${{ github.sha }} IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} run: | - docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest . + docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest -f ./proxy/Dockerfile ./proxy docker push $IMAGE_URI:$IMAGE_TAG docker push $IMAGE_URI:latest echo "pushed proxy image" \ No newline at end of file From 8f3efe4b26681eae7db1b29edd6d0fe955407851 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 11:12:43 -0800 Subject: [PATCH 52/98] use multiarchitechure builds in gh action --- .github/workflows/deploy-ec2-staging.yml | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 0a40ecb1..a51d1d3f 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -27,15 +27,21 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build, tag, and push web Docker image to ECR id: build-web-image env: IMAGE_TAG: ${{ github.sha }} IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} run: | - docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest -f ./TEKDB/Dockerfile ./TEKDB - docker push $IMAGE_URI:$IMAGE_TAG - docker push $IMAGE_URI:latest + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./TEKDB/Dockerfile ./TEKDB \ + --push echo "pushed web image" - name: Build, tag, and push proxy Docker image to ECR @@ -44,7 +50,10 @@ jobs: IMAGE_TAG: ${{ github.sha }} IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} run: | - docker build -t $IMAGE_URI:$IMAGE_TAG -t $IMAGE_URI:latest -f ./proxy/Dockerfile ./proxy - docker push $IMAGE_URI:$IMAGE_TAG - docker push $IMAGE_URI:latest + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./proxy/Dockerfile ./proxy \ + --push echo "pushed proxy image" \ No newline at end of file From 3a8e9ab5a20f797b804340f9f6ae5a54c3023f24 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 13:43:44 -0800 Subject: [PATCH 53/98] wip: use ssm to deploy docker --- .github/workflows/deploy-ec2-staging.yml | 115 +++++++++++++---------- infra/iam.tf | 5 + 2 files changed, 72 insertions(+), 48 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index a51d1d3f..b17f73dc 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -7,53 +7,72 @@ on: workflow_dispatch: jobs: - build-push-and-deploy: - name: Build, Push to ECR, Deploy to EC2 Staging - runs-on: ubuntu-latest - permissions: - id-token: write + build-push-and-deploy: + name: Build, Push to ECR, Deploy to EC2 Staging + runs-on: ubuntu-latest + permissions: + id-token: write - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v5 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION}} - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v5 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION}} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Build, tag, and push web Docker image to ECR - id: build-web-image - env: - IMAGE_TAG: ${{ github.sha }} - IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t $IMAGE_URI:$IMAGE_TAG \ - -t $IMAGE_URI:latest \ - -f ./TEKDB/Dockerfile ./TEKDB \ - --push - echo "pushed web image" - - - name: Build, tag, and push proxy Docker image to ECR - id: build-proxy-image - env: - IMAGE_TAG: ${{ github.sha }} - IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t $IMAGE_URI:$IMAGE_TAG \ - -t $IMAGE_URI:latest \ - -f ./proxy/Dockerfile ./proxy \ - --push - echo "pushed proxy image" \ No newline at end of file + - name: Build, tag, and push web Docker image to ECR + id: build-web-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./TEKDB/Dockerfile ./TEKDB \ + --push + + - name: Build, tag, and push proxy Docker image to ECR + id: build-proxy-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./proxy/Dockerfile ./proxy \ + --push + - name: get EC2 Instance ID + id: get-instance + run: | + INSTANCE_ID=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=${{ secrets.EC2_INSTANCE_NAME }}" \ + "Name=instance-state-name,Values=running" \ + --query "Reservations[0].Instances[0].InstanceId" \ + --output text) + - name: Deploy via SSM + run: | + aws ssm send-command \ + --instance-ids "${{ steps.get-instance.outputs.instance_id }}" \ + --document-name "AWS-RunShellScript" \ + --parameters 'commands=[ + "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username ${{ secrets.AWS_USER }} --password-stdin $(echo \"${{ secrets.WEB_ECR_IMAGE_URI }}", + "sudo docker pull ${{ secrets.ECR_IMAGE_URI_APP }}:latest", + "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username ${{ secrets.AWS_USER }} --password-stdin $(echo \"${{ secrets.PROXY_ECR_IMAGE_URI }}" + "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", + "cd /TEKDB", + "sudo docker compose down", + "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d" diff --git a/infra/iam.tf b/infra/iam.tf index fcf46649..368352fb 100644 --- a/infra/iam.tf +++ b/infra/iam.tf @@ -21,6 +21,11 @@ resource "aws_iam_role_policy_attachment" "ecr_read" { policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" } +resource "aws_iam_role_policy_attachment" "ssm_core" { + role = aws_iam_role.ec2_ecr_role.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + # Instance profile wraps the role so EC2 can use it resource "aws_iam_instance_profile" "ec2_profile" { name = "${var.project_name}-ec2-profile" From 87c5b218438dd2c1323b88bf98575477bd8cda7c Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 13:49:33 -0800 Subject: [PATCH 54/98] only manually trigger create-and-publish-docker-images gh action --- .github/workflows/create-and-publish-docker-images.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 3a0bf021..c7818a09 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -3,8 +3,7 @@ name: Create and publish a Docker image # manually trigger while testing on: - push: - + workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: From 130d0fb93d02d972db8961a60ec27bed9570719f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 14:11:00 -0800 Subject: [PATCH 55/98] wip: comment out build step, fix syntax in ssm commands --- .github/workflows/deploy-ec2-staging.yml | 56 ++++++++++++------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index b17f73dc..14062e70 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -30,31 +30,31 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build, tag, and push web Docker image to ECR - id: build-web-image - env: - IMAGE_TAG: ${{ github.sha }} - IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t $IMAGE_URI:$IMAGE_TAG \ - -t $IMAGE_URI:latest \ - -f ./TEKDB/Dockerfile ./TEKDB \ - --push + # - name: Build, tag, and push web Docker image to ECR + # id: build-web-image + # env: + # IMAGE_TAG: ${{ github.sha }} + # IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} + # run: | + # docker buildx build \ + # --platform linux/amd64,linux/arm64 \ + # -t $IMAGE_URI:$IMAGE_TAG \ + # -t $IMAGE_URI:latest \ + # -f ./TEKDB/Dockerfile ./TEKDB \ + # --push - - name: Build, tag, and push proxy Docker image to ECR - id: build-proxy-image - env: - IMAGE_TAG: ${{ github.sha }} - IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} - run: | - docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -t $IMAGE_URI:$IMAGE_TAG \ - -t $IMAGE_URI:latest \ - -f ./proxy/Dockerfile ./proxy \ - --push + # - name: Build, tag, and push proxy Docker image to ECR + # id: build-proxy-image + # env: + # IMAGE_TAG: ${{ github.sha }} + # IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} + # run: | + # docker buildx build \ + # --platform linux/amd64,linux/arm64 \ + # -t $IMAGE_URI:$IMAGE_TAG \ + # -t $IMAGE_URI:latest \ + # -f ./proxy/Dockerfile ./proxy \ + # --push - name: get EC2 Instance ID id: get-instance run: | @@ -63,16 +63,18 @@ jobs: "Name=instance-state-name,Values=running" \ --query "Reservations[0].Instances[0].InstanceId" \ --output text) + echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT - name: Deploy via SSM run: | aws ssm send-command \ --instance-ids "${{ steps.get-instance.outputs.instance_id }}" \ --document-name "AWS-RunShellScript" \ --parameters 'commands=[ - "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username ${{ secrets.AWS_USER }} --password-stdin $(echo \"${{ secrets.WEB_ECR_IMAGE_URI }}", - "sudo docker pull ${{ secrets.ECR_IMAGE_URI_APP }}:latest", - "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username ${{ secrets.AWS_USER }} --password-stdin $(echo \"${{ secrets.PROXY_ECR_IMAGE_URI }}" + "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.WEB_ECR_IMAGE_URI }}", + "sudo docker pull ${{ secrets.WEB_ECR_IMAGE_URI }}:latest", + "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.PROXY_ECR_IMAGE_URI }}", "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", "cd /TEKDB", "sudo docker compose down", "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d" + ]' From 3513e94c1082b21919fee8b6107e03b00cd10c50 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 14:18:13 -0800 Subject: [PATCH 56/98] try verifying deployment status --- .github/workflows/deploy-ec2-staging.yml | 31 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 14062e70..0201e004 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -65,8 +65,9 @@ jobs: --output text) echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT - name: Deploy via SSM + id: ssm-deploy run: | - aws ssm send-command \ + COMMAND_ID=$(aws ssm send-command \ --instance-ids "${{ steps.get-instance.outputs.instance_id }}" \ --document-name "AWS-RunShellScript" \ --parameters 'commands=[ @@ -76,5 +77,29 @@ jobs: "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", "cd /TEKDB", "sudo docker compose down", - "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d" - ]' + "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d -p tekdb" + ]' \ + --timeout-seconds 120 \ + --comment "Deploy ${{ github.sha }}" \ + --query "Command.CommandId" \ + --output text) + echo "command_id=$COMMAND_ID" >> $GITHUB_OUTPUT + - name: Wait for deployment to complete + run: | + aws ssm wait command-executed \ + --command-id ${{ steps.ssm-deploy.outputs.command_id }} \ + --instance-id ${{ steps.get-instance.outputs.instance_id }} + - name: Verify deployment result + run: | + RESULT=$(aws ssm get-command-invocation \ + --command-id ${{ steps.ssm-deploy.outputs.command_id }} \ + --instance-id ${{ steps.get-instance.outputs.instance_id }} \ + --query "{Status:Status,Output:StandardOutputContent,Error:StandardErrorContent}" \ + --output json) + echo "$RESULT" | jq . + STATUS=$(echo "$RESULT" | jq -r '.Status') + if [ "$STATUS" != "Success" ]; then + echo "::error::Deployment failed with status: $STATUS" + exit 1 + fi + echo "Deployment succeeded." From 0f25d03c967a3989e10fb4ce4aa0ee3fd9c3e4b4 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 14:39:40 -0800 Subject: [PATCH 57/98] remove -p from docker compose up --- .github/workflows/deploy-ec2-staging.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 0201e004..817c4261 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -77,7 +77,9 @@ jobs: "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", "cd /TEKDB", "sudo docker compose down", - "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d -p tekdb" + "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d", + "sleep 5", + "docker ps --filter name=web --format \"{{.Status}}\"" ]' \ --timeout-seconds 120 \ --comment "Deploy ${{ github.sha }}" \ From 2c28abdcb1c2dfb87b33617989d53f9f19a46c3d Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 3 Mar 2026 14:45:49 -0800 Subject: [PATCH 58/98] comment back in docker build image steps --- .github/workflows/deploy-ec2-staging.yml | 49 ++++++++++++------------ 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 817c4261..638c07f5 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -1,7 +1,6 @@ name: Build, Push, and Deploy to EC2 Staging on: - push: release: types: [published] workflow_dispatch: @@ -30,31 +29,31 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # - name: Build, tag, and push web Docker image to ECR - # id: build-web-image - # env: - # IMAGE_TAG: ${{ github.sha }} - # IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} - # run: | - # docker buildx build \ - # --platform linux/amd64,linux/arm64 \ - # -t $IMAGE_URI:$IMAGE_TAG \ - # -t $IMAGE_URI:latest \ - # -f ./TEKDB/Dockerfile ./TEKDB \ - # --push + - name: Build, tag, and push web Docker image to ECR + id: build-web-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./TEKDB/Dockerfile ./TEKDB \ + --push - # - name: Build, tag, and push proxy Docker image to ECR - # id: build-proxy-image - # env: - # IMAGE_TAG: ${{ github.sha }} - # IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} - # run: | - # docker buildx build \ - # --platform linux/amd64,linux/arm64 \ - # -t $IMAGE_URI:$IMAGE_TAG \ - # -t $IMAGE_URI:latest \ - # -f ./proxy/Dockerfile ./proxy \ - # --push + - name: Build, tag, and push proxy Docker image to ECR + id: build-proxy-image + env: + IMAGE_TAG: ${{ github.sha }} + IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }} + run: | + docker buildx build \ + --platform linux/amd64,linux/arm64 \ + -t $IMAGE_URI:$IMAGE_TAG \ + -t $IMAGE_URI:latest \ + -f ./proxy/Dockerfile ./proxy \ + --push - name: get EC2 Instance ID id: get-instance run: | From b3e3dc10ecdb0f14e825b6c82a5cd1949c48180a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Mar 2026 13:38:53 -0800 Subject: [PATCH 59/98] add logs module to track expired chunk deletion --- TEKDB/Logs/__init__.py | 2 ++ TEKDB/Logs/admin.py | 27 ++++++++++++++++++++++ TEKDB/Logs/apps.py | 6 +++++ TEKDB/Logs/migrations/0001_initial.py | 33 +++++++++++++++++++++++++++ TEKDB/Logs/migrations/__init__.py | 0 TEKDB/Logs/models.py | 17 ++++++++++++++ TEKDB/Logs/tests/__init__.py | 0 TEKDB/Logs/tests/test_forms.py | 0 TEKDB/Logs/tests/test_models.py | 0 TEKDB/Logs/tests/test_views.py | 0 TEKDB/Logs/views.py | 1 + TEKDB/TEKDB/settings.py | 1 + 12 files changed, 87 insertions(+) create mode 100644 TEKDB/Logs/__init__.py create mode 100644 TEKDB/Logs/admin.py create mode 100644 TEKDB/Logs/apps.py create mode 100644 TEKDB/Logs/migrations/0001_initial.py create mode 100644 TEKDB/Logs/migrations/__init__.py create mode 100644 TEKDB/Logs/models.py create mode 100644 TEKDB/Logs/tests/__init__.py create mode 100644 TEKDB/Logs/tests/test_forms.py create mode 100644 TEKDB/Logs/tests/test_models.py create mode 100644 TEKDB/Logs/tests/test_views.py create mode 100644 TEKDB/Logs/views.py diff --git a/TEKDB/Logs/__init__.py b/TEKDB/Logs/__init__.py new file mode 100644 index 00000000..97e929b4 --- /dev/null +++ b/TEKDB/Logs/__init__.py @@ -0,0 +1,2 @@ +# TEKDB/__init__.py + diff --git a/TEKDB/Logs/admin.py b/TEKDB/Logs/admin.py new file mode 100644 index 00000000..995fda03 --- /dev/null +++ b/TEKDB/Logs/admin.py @@ -0,0 +1,27 @@ +from django.contrib import admin +from Logs.models import ExpiredChunkDeletionLog + + +class ExpiredChunkDeletionLogAdmin(admin.ModelAdmin): + readonly_fields = [ + "file_name", + "file_path", + "file_size", + "original_created_at", + "deleted_at", + "reason", + "success", + "error_message", + ] + + def has_add_permission(self, request): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False + + +admin.site.register(ExpiredChunkDeletionLog, ExpiredChunkDeletionLogAdmin) diff --git a/TEKDB/Logs/apps.py b/TEKDB/Logs/apps.py new file mode 100644 index 00000000..2ca2e123 --- /dev/null +++ b/TEKDB/Logs/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class LogsConfig(AppConfig): + name = "Logs" + verbose_name = "Logs" diff --git a/TEKDB/Logs/migrations/0001_initial.py b/TEKDB/Logs/migrations/0001_initial.py new file mode 100644 index 00000000..3ba9a6e7 --- /dev/null +++ b/TEKDB/Logs/migrations/0001_initial.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.29 on 2026-03-04 21:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='MediaFileDeletionLog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('file_name', models.CharField(max_length=255)), + ('file_path', models.CharField(max_length=500)), + ('file_size', models.CharField(max_length=50, null=True)), + ('original_created_at', models.DateTimeField()), + ('deleted_at', models.DateTimeField(auto_now_add=True)), + ('reason', models.CharField(default='age_policy', max_length=100)), + ('success', models.BooleanField(default=True)), + ('error_message', models.TextField(blank=True)), + ], + options={ + 'verbose_name': 'Partial Upload Deletion Log', + 'verbose_name_plural': 'Partial Upload Deletion Logs', + 'indexes': [models.Index(fields=['deleted_at'], name='Logs_mediaf_deleted_17c511_idx')], + }, + ), + ] diff --git a/TEKDB/Logs/migrations/__init__.py b/TEKDB/Logs/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TEKDB/Logs/models.py b/TEKDB/Logs/models.py new file mode 100644 index 00000000..b33a85fc --- /dev/null +++ b/TEKDB/Logs/models.py @@ -0,0 +1,17 @@ +from django.db import models + + +class ExpiredChunkDeletionLog(models.Model): + file_name = models.CharField(max_length=255) + file_path = models.CharField(max_length=500) + file_size = models.CharField(max_length=50, null=True) + original_created_at = models.DateTimeField() + deleted_at = models.DateTimeField(auto_now_add=True) + reason = models.CharField(max_length=100, default="age_policy") + success = models.BooleanField(default=True) + error_message = models.TextField(blank=True) + + class Meta: + indexes = [models.Index(fields=["deleted_at"])] + verbose_name = "Partial Upload Deletion Log" + verbose_name_plural = "Partial Upload Deletion Logs" diff --git a/TEKDB/Logs/tests/__init__.py b/TEKDB/Logs/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/TEKDB/Logs/tests/test_forms.py b/TEKDB/Logs/tests/test_forms.py new file mode 100644 index 00000000..e69de29b diff --git a/TEKDB/Logs/tests/test_models.py b/TEKDB/Logs/tests/test_models.py new file mode 100644 index 00000000..e69de29b diff --git a/TEKDB/Logs/tests/test_views.py b/TEKDB/Logs/tests/test_views.py new file mode 100644 index 00000000..e69de29b diff --git a/TEKDB/Logs/views.py b/TEKDB/Logs/views.py new file mode 100644 index 00000000..60f00ef0 --- /dev/null +++ b/TEKDB/Logs/views.py @@ -0,0 +1 @@ +# Create your views here. diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 49af92a8..b485f5ca 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -68,6 +68,7 @@ "Lookup", "Accounts", "Relationships", + "Logs", "reversion", "django.contrib.sites", "django_resumable_async_upload", From b943f3ba2cd100bec36d23a0688c18e8512a7da0 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Mar 2026 13:39:46 -0800 Subject: [PATCH 60/98] management command to delete expired chunks --- .../commands/delete_expired_chunks.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 TEKDB/TEKDB/management/commands/delete_expired_chunks.py diff --git a/TEKDB/TEKDB/management/commands/delete_expired_chunks.py b/TEKDB/TEKDB/management/commands/delete_expired_chunks.py new file mode 100644 index 00000000..7127303f --- /dev/null +++ b/TEKDB/TEKDB/management/commands/delete_expired_chunks.py @@ -0,0 +1,108 @@ +import os +import logging +from datetime import timedelta + +from django.core.management.base import BaseCommand +from django.utils import timezone + +logger = logging.getLogger("delete_partial_upload") + + +def bytes_to_readable(num_bytes, suffix="B"): + """Converts bytes to a human-readable format (e.g., KB, MB, GB).""" + for unit in ["", "K", "M", "G", "T", "P"]: + if num_bytes < 1024: + return f"{num_bytes:.2f} {unit}{suffix}" + num_bytes /= 1024 + + +class Command(BaseCommand): + help = "Delete interrupted uploads files older than X hours from a specific media subfolder" + + def add_arguments(self, parser): + parser.add_argument( + "--hours", type=int, default=24, help="Max file age in hours" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Preview deletions without actually deleting", + ) + + def handle(self, *args, **options): + from django.conf import settings + from Logs.models import ExpiredChunkDeletionLog + + target_dir = os.path.join( + settings.MEDIA_ROOT, settings.ADMIN_RESUMABLE_CHUNK_FOLDER + ) + max_age_hours = options["hours"] + dry_run = options["dry_run"] + cutoff = timezone.now() - timedelta(hours=max_age_hours) + cutoff_timestamp = cutoff.timestamp() + + if not os.path.isdir(target_dir): + logger.error(f"Target directory does not exist: {target_dir}") + self.stderr.write(f"Directory not found: {target_dir}") + return + + logger.info( + f"Starting cleanup of '{target_dir}' — files older than {max_age_hours}h (dry_run={dry_run})" + ) + + deleted, failed, skipped = [], [], [] + for root, dirs, files in os.walk(target_dir): + for filename in files: + file_path = os.path.join(root, filename) + try: + mtime = os.path.getmtime(file_path) + if mtime >= cutoff_timestamp: + skipped.append(file_path) + continue + + file_size_bytes = os.path.getsize(file_path) + file_size = bytes_to_readable(file_size_bytes) + file_age_hours = (timezone.now().timestamp() - mtime) / 3600 + logger.info( + f"{'[DRY RUN] Would delete' if dry_run else 'Deleting'}: {file_path} " + f"(size={file_size}, age={file_age_hours:.1f}h)" + ) + + if not dry_run: + os.remove(file_path) + ExpiredChunkDeletionLog.objects.create( + file_name=filename, + file_path=file_path, + file_size=file_size, + original_created_at=timezone.datetime.fromtimestamp( + mtime, tz=timezone.utc + ), + reason="age_policy", + success=True, + ) + deleted.append(file_path) + + except Exception as e: + logger.error(f"Failed to delete {file_path}: {e}") + if not dry_run: + ExpiredChunkDeletionLog.objects.create( + file_name=filename, + file_path=file_path, + file_size=file_size, + original_created_at=timezone.datetime.fromtimestamp( + mtime, tz=timezone.utc + ), + reason="age_policy", + success=False, + error_message=str(e), + ) + failed.append(file_path) + + logger.info( + f"Cleanup complete — deleted: {len(deleted)}, failed: {len(failed)}, skipped (too new): {len(skipped)}" + ) + self.stdout.write( + self.style.SUCCESS( + f"Deletion completed at {timezone.now()}. Deleted: {len(deleted)}, Failed: {len(failed)}, Skipped: {len(skipped)}" + ) + ) From 6ccc0ca499c135a5fb19a4467c2f7eec3faf860b Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Mar 2026 13:45:00 -0800 Subject: [PATCH 61/98] ruff format --- TEKDB/TEKDB/tekdb_filebrowser.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/TEKDB/TEKDB/tekdb_filebrowser.py b/TEKDB/TEKDB/tekdb_filebrowser.py index d166e6d2..dc4420b6 100644 --- a/TEKDB/TEKDB/tekdb_filebrowser.py +++ b/TEKDB/TEKDB/tekdb_filebrowser.py @@ -184,8 +184,10 @@ def delete_media_without_record_confirm(self, request): filelisting = self.filelisting_class( path, - filter_func=lambda fo: not fo.has_media_record() - and fo.filename not in self.files_folders_to_ignore(), + filter_func=lambda fo: ( + not fo.has_media_record() + and fo.filename not in self.files_folders_to_ignore() + ), sorting_by=query.get("o", DEFAULT_SORTING_BY), sorting_order=query.get("ot", DEFAULT_SORTING_ORDER), site=self, @@ -216,8 +218,10 @@ def delete_media_without_record(self, request): filelisting = self.filelisting_class( path, - filter_func=lambda fo: not fo.has_media_record() - and fo.filename not in self.files_folders_to_ignore(), + filter_func=lambda fo: ( + not fo.has_media_record() + and fo.filename not in self.files_folders_to_ignore() + ), sorting_by=query.get("o", DEFAULT_SORTING_BY), sorting_order=query.get("ot", DEFAULT_SORTING_ORDER), site=self, From 499517b8735d9c42f2c907fa1a991299efee2bc3 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 4 Mar 2026 14:48:23 -0800 Subject: [PATCH 62/98] remove dynamic part of defaultmediabulkname --- ...0027_alter_mediabulkupload_mediabulkname.py | 18 ++++++++++++++++++ TEKDB/TEKDB/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 TEKDB/TEKDB/migrations/0027_alter_mediabulkupload_mediabulkname.py diff --git a/TEKDB/TEKDB/migrations/0027_alter_mediabulkupload_mediabulkname.py b/TEKDB/TEKDB/migrations/0027_alter_mediabulkupload_mediabulkname.py new file mode 100644 index 00000000..875df594 --- /dev/null +++ b/TEKDB/TEKDB/migrations/0027_alter_mediabulkupload_mediabulkname.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.29 on 2026-03-04 22:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('TEKDB', '0026_alter_media_mediafile_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='mediabulkupload', + name='mediabulkname', + field=models.CharField(blank=True, default='Bulk Media Upload', max_length=255, null=True, verbose_name='name'), + ), + ] diff --git a/TEKDB/TEKDB/models.py b/TEKDB/TEKDB/models.py index f91e20fb..4f12cf6e 100644 --- a/TEKDB/TEKDB/models.py +++ b/TEKDB/TEKDB/models.py @@ -2696,7 +2696,7 @@ class MediaBulkUpload(Reviewable, Queryable, Record, ModeratedModel): from datetime import date # Default name for the media bulk upload - defaultmediabulkname = "Bulk Upload on %s" % date.today() + defaultmediabulkname = "Bulk Media Upload" mediabulkname = models.CharField( max_length=255, From 61aa1d92a039826b0b895f4fde6edff842b419f4 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 11:14:25 -0800 Subject: [PATCH 63/98] wip: add celery for deleting expired chunks --- TEKDB/Logs/tasks.py | 92 +++++++++++++++ TEKDB/TEKDB/__init__.py | 4 + TEKDB/TEKDB/celery.py | 22 ++++ .../commands/delete_expired_chunks.py | 108 ------------------ TEKDB/TEKDB/settings.py | 10 ++ TEKDB/media/__init__.py | 0 TEKDB/requirements.txt | 3 + 7 files changed, 131 insertions(+), 108 deletions(-) create mode 100644 TEKDB/Logs/tasks.py create mode 100644 TEKDB/TEKDB/celery.py mode change 100755 => 100644 TEKDB/media/__init__.py diff --git a/TEKDB/Logs/tasks.py b/TEKDB/Logs/tasks.py new file mode 100644 index 00000000..8ed8edcd --- /dev/null +++ b/TEKDB/Logs/tasks.py @@ -0,0 +1,92 @@ +import os +import logging +from datetime import timedelta +from celery import shared_task + +from django.utils import timezone + +logger = logging.getLogger("delete_partial_upload") + + +def bytes_to_readable(num_bytes, suffix="B"): + """Converts bytes to a human-readable format (e.g., KB, MB, GB).""" + for unit in ["", "K", "M", "G", "T", "P"]: + if num_bytes < 1024: + return f"{num_bytes:.2f} {unit}{suffix}" + num_bytes /= 1024 + + +@shared_task(bind=True, max_retries=3, autoretry_for=(Exception,)) +def delete_old_media_files(self, max_age_hours=24): + + from django.conf import settings + from Logs.models import ExpiredChunkDeletionLog + + target_dir = os.path.join( + settings.MEDIA_ROOT, settings.ADMIN_RESUMABLE_CHUNK_FOLDER + ) + cutoff = timezone.now() - timedelta(hours=max_age_hours) + cutoff_timestamp = cutoff.timestamp() + + if not os.path.isdir(target_dir): + logger.error(f"Target directory does not exist: {target_dir}") + return + + logger.info( + f"Starting cleanup of '{target_dir}' — files older than {max_age_hours}h" + ) + + deleted, failed, skipped = [], [], [] + for root, dirs, files in os.walk(target_dir): + for filename in files: + file_path = os.path.join(root, filename) + try: + mtime = os.path.getmtime(file_path) + if mtime >= cutoff_timestamp: + skipped.append(file_path) + continue + + file_size_bytes = os.path.getsize(file_path) + file_size = bytes_to_readable(file_size_bytes) + file_age_hours = (timezone.now().timestamp() - mtime) / 3600 + logger.info( + f"Deleting: {file_path} " + f"(size={file_size}, age={file_age_hours:.1f}h)" + ) + + os.remove(file_path) + ExpiredChunkDeletionLog.objects.create( + file_name=filename, + file_path=file_path, + file_size=file_size, + original_created_at=timezone.datetime.fromtimestamp( + mtime, tz=timezone.utc + ), + reason="age_policy", + success=True, + ) + deleted.append(file_path) + + except Exception as e: + logger.error(f"Failed to delete {file_path}: {e}") + ExpiredChunkDeletionLog.objects.create( + file_name=filename, + file_path=file_path, + file_size=file_size, + original_created_at=timezone.datetime.fromtimestamp( + mtime, tz=timezone.utc + ), + reason="age_policy", + success=False, + error_message=str(e), + ) + failed.append(file_path) + + logger.info( + f"Cleanup complete — deleted: {len(deleted)}, failed: {len(failed)}, skipped (too new): {len(skipped)}" + ) + self.stdout.write( + self.style.SUCCESS( + f"Deletion completed at {timezone.now()}. Deleted: {len(deleted)}, Failed: {len(failed)}, Skipped: {len(skipped)}" + ) + ) diff --git a/TEKDB/TEKDB/__init__.py b/TEKDB/TEKDB/__init__.py index 97e929b4..7fa5c800 100644 --- a/TEKDB/TEKDB/__init__.py +++ b/TEKDB/TEKDB/__init__.py @@ -1,2 +1,6 @@ # TEKDB/__init__.py +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app +__all__ = ("celery_app",) diff --git a/TEKDB/TEKDB/celery.py b/TEKDB/TEKDB/celery.py new file mode 100644 index 00000000..0f1cbcc4 --- /dev/null +++ b/TEKDB/TEKDB/celery.py @@ -0,0 +1,22 @@ +import os + +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "TEKDB.settings") + +app = Celery("TEKDB") + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f"Request: {self.request!r}") diff --git a/TEKDB/TEKDB/management/commands/delete_expired_chunks.py b/TEKDB/TEKDB/management/commands/delete_expired_chunks.py index 7127303f..e69de29b 100644 --- a/TEKDB/TEKDB/management/commands/delete_expired_chunks.py +++ b/TEKDB/TEKDB/management/commands/delete_expired_chunks.py @@ -1,108 +0,0 @@ -import os -import logging -from datetime import timedelta - -from django.core.management.base import BaseCommand -from django.utils import timezone - -logger = logging.getLogger("delete_partial_upload") - - -def bytes_to_readable(num_bytes, suffix="B"): - """Converts bytes to a human-readable format (e.g., KB, MB, GB).""" - for unit in ["", "K", "M", "G", "T", "P"]: - if num_bytes < 1024: - return f"{num_bytes:.2f} {unit}{suffix}" - num_bytes /= 1024 - - -class Command(BaseCommand): - help = "Delete interrupted uploads files older than X hours from a specific media subfolder" - - def add_arguments(self, parser): - parser.add_argument( - "--hours", type=int, default=24, help="Max file age in hours" - ) - parser.add_argument( - "--dry-run", - action="store_true", - help="Preview deletions without actually deleting", - ) - - def handle(self, *args, **options): - from django.conf import settings - from Logs.models import ExpiredChunkDeletionLog - - target_dir = os.path.join( - settings.MEDIA_ROOT, settings.ADMIN_RESUMABLE_CHUNK_FOLDER - ) - max_age_hours = options["hours"] - dry_run = options["dry_run"] - cutoff = timezone.now() - timedelta(hours=max_age_hours) - cutoff_timestamp = cutoff.timestamp() - - if not os.path.isdir(target_dir): - logger.error(f"Target directory does not exist: {target_dir}") - self.stderr.write(f"Directory not found: {target_dir}") - return - - logger.info( - f"Starting cleanup of '{target_dir}' — files older than {max_age_hours}h (dry_run={dry_run})" - ) - - deleted, failed, skipped = [], [], [] - for root, dirs, files in os.walk(target_dir): - for filename in files: - file_path = os.path.join(root, filename) - try: - mtime = os.path.getmtime(file_path) - if mtime >= cutoff_timestamp: - skipped.append(file_path) - continue - - file_size_bytes = os.path.getsize(file_path) - file_size = bytes_to_readable(file_size_bytes) - file_age_hours = (timezone.now().timestamp() - mtime) / 3600 - logger.info( - f"{'[DRY RUN] Would delete' if dry_run else 'Deleting'}: {file_path} " - f"(size={file_size}, age={file_age_hours:.1f}h)" - ) - - if not dry_run: - os.remove(file_path) - ExpiredChunkDeletionLog.objects.create( - file_name=filename, - file_path=file_path, - file_size=file_size, - original_created_at=timezone.datetime.fromtimestamp( - mtime, tz=timezone.utc - ), - reason="age_policy", - success=True, - ) - deleted.append(file_path) - - except Exception as e: - logger.error(f"Failed to delete {file_path}: {e}") - if not dry_run: - ExpiredChunkDeletionLog.objects.create( - file_name=filename, - file_path=file_path, - file_size=file_size, - original_created_at=timezone.datetime.fromtimestamp( - mtime, tz=timezone.utc - ), - reason="age_policy", - success=False, - error_message=str(e), - ) - failed.append(file_path) - - logger.info( - f"Cleanup complete — deleted: {len(deleted)}, failed: {len(failed)}, skipped (too new): {len(skipped)}" - ) - self.stdout.write( - self.style.SUCCESS( - f"Deletion completed at {timezone.now()}. Deleted: {len(deleted)}, Failed: {len(failed)}, Skipped: {len(skipped)}" - ) - ) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index b485f5ca..49b0c119 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -55,6 +55,8 @@ "django.contrib.messages", "django.contrib.staticfiles", "django.contrib.gis", + "django_celery_results", + "django_celery_beat", "colorfield", # 'registration', "leaflet", @@ -353,6 +355,14 @@ ADMIN_SIMULTANEOUS_UPLOADS = 1 ADMIN_RESUMABLE_CHUNK_FOLDER = "resumable_chunks" +# Celery settings + +# Celery Configuration Options +CELERY_TIMEZONE = TIME_ZONE +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CELERY_RESULT_BACKEND = "django-db" + try: from TEKDB.local_settings import * # noqa: F403 except Exception: diff --git a/TEKDB/media/__init__.py b/TEKDB/media/__init__.py old mode 100755 new mode 100644 diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 9a0cf228..782855a5 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -1,6 +1,7 @@ #REQUIRED confusable-homoglyphs coverage +celery django >=4.2.16,<4.3 django-autocomplete-light django-ckeditor @@ -10,6 +11,8 @@ django-nested-admin django-registration django-reversion django-tinymce +django-celery-results +django-celery-beat pillow psycopg2-binary psutil From 49ddfa53e62bceb0eb64b939a4101e1a5abc4684 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 11:16:46 -0800 Subject: [PATCH 64/98] trigger ec2 on push to develop branch --- .github/workflows/deploy-ec2-staging.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 638c07f5..55e4eeb6 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -1,8 +1,9 @@ name: Build, Push, and Deploy to EC2 Staging on: - release: - types: [published] + push: + branches: + - develop workflow_dispatch: jobs: From ac50543dd2ed851e2d33d6bc7c5d4c5b6ec52740 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 11:20:34 -0800 Subject: [PATCH 65/98] push to GHR on merge to main --- .github/workflows/create-and-publish-docker-images.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index c7818a09..99d9467f 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -3,6 +3,9 @@ name: Create and publish a Docker image # manually trigger while testing on: + push: + branches: + - develop workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. From 4df94c85d9e4562927f6122f8489723bb31046a7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 11:21:06 -0800 Subject: [PATCH 66/98] push to GHR on merge to main --- .github/workflows/create-and-publish-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 99d9467f..0b177d93 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -5,7 +5,7 @@ name: Create and publish a Docker image on: push: branches: - - develop + - main workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. From 8cecf62dc82bb8b8206d7043979614ca97f5f3bb Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 12:58:08 -0800 Subject: [PATCH 67/98] sequence containers, update env vars --- TEKDB/TEKDB/settings.py | 3 +- TEKDB/requirements.txt | 1 + docker/docker-compose.yml | 67 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 49b0c119..c9b9028f 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -355,12 +355,11 @@ ADMIN_SIMULTANEOUS_UPLOADS = 1 ADMIN_RESUMABLE_CHUNK_FOLDER = "resumable_chunks" -# Celery settings - # Celery Configuration Options CELERY_TIMEZONE = TIME_ZONE CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 30 * 60 +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0") CELERY_RESULT_BACKEND = "django-db" try: diff --git a/TEKDB/requirements.txt b/TEKDB/requirements.txt index 782855a5..4cbff22c 100644 --- a/TEKDB/requirements.txt +++ b/TEKDB/requirements.txt @@ -16,6 +16,7 @@ django-celery-beat pillow psycopg2-binary psutil +redis django-filebrowser-no-grappelli>=4.0.0,<5.0.0 XlsxWriter django-resumable-async-upload diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f865e83a..85f77c48 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -23,7 +23,10 @@ services: dockerfile: ../TEKDB/Dockerfile restart: unless-stopped depends_on: - - db + db: + condition: service_healthy + redis: + condition: service_started env_file: - .env.dev environment: @@ -36,10 +39,72 @@ services: SQL_USER: ${SQL_USER} SQL_PASSWORD: ${SQL_PASSWORD} SECRET_KEY: ${SECRET_KEY} + CELERY_BROKER_URL: redis://redis:6379/0 ports: - "8000:8000" volumes: - ../TEKDB:/usr/src/app + healthcheck: + test: ["CMD-SHELL", "python manage.py check --deploy 2>/dev/null || python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:8000/\")' 2>/dev/null || exit 0"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + redis: + image: redis:alpine + + celery: + build: ../TEKDB/ + entrypoint: [] + command: celery -A TEKDB worker -l info + volumes: + - ../TEKDB:/usr/src/app + env_file: + - .env.dev + environment: + ALLOWED_HOSTS: ${ALLOWED_HOSTS} + DEBUG: ${DEBUG} + SQL_ENGINE: ${SQL_ENGINE} + SQL_HOST: ${SQL_HOST} + SQL_PORT: ${SQL_PORT} + SQL_DATABASE: ${SQL_DATABASE} + SQL_USER: ${SQL_USER} + SQL_PASSWORD: ${SQL_PASSWORD} + SECRET_KEY: ${SECRET_KEY} + CELERY_BROKER_URL: redis://redis:6379/0 + depends_on: + web: + condition: service_healthy + db: + condition: service_healthy + redis: + condition: service_started + celery-beat: + build: ../TEKDB/ + entrypoint: [] + command: celery -A TEKDB beat -l info + volumes: + - ../TEKDB:/usr/src/app + env_file: + - .env.dev + environment: + ALLOWED_HOSTS: ${ALLOWED_HOSTS} + DEBUG: ${DEBUG} + SQL_ENGINE: ${SQL_ENGINE} + SQL_HOST: ${SQL_HOST} + SQL_PORT: ${SQL_PORT} + SQL_DATABASE: ${SQL_DATABASE} + SQL_USER: ${SQL_USER} + SQL_PASSWORD: ${SQL_PASSWORD} + SECRET_KEY: ${SECRET_KEY} + CELERY_BROKER_URL: redis://redis:6379/0 + depends_on: + web: + condition: service_healthy + db: + condition: service_healthy + redis: + condition: service_started volumes: tekdb_db_data: From 7e4b7b97d84f844d0fc0215931163e05bc28a78c Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 13:25:38 -0800 Subject: [PATCH 68/98] schedule task to delete remaining chunks --- TEKDB/Logs/tasks.py | 11 ++++++----- TEKDB/TEKDB/settings.py | 9 +++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/TEKDB/Logs/tasks.py b/TEKDB/Logs/tasks.py index 8ed8edcd..14594123 100644 --- a/TEKDB/Logs/tasks.py +++ b/TEKDB/Logs/tasks.py @@ -85,8 +85,9 @@ def delete_old_media_files(self, max_age_hours=24): logger.info( f"Cleanup complete — deleted: {len(deleted)}, failed: {len(failed)}, skipped (too new): {len(skipped)}" ) - self.stdout.write( - self.style.SUCCESS( - f"Deletion completed at {timezone.now()}. Deleted: {len(deleted)}, Failed: {len(failed)}, Skipped: {len(skipped)}" - ) - ) + return { + "deleted": len(deleted), + "failed": len(failed), + "skipped": len(skipped), + "completed_at": timezone.now().isoformat(), + } diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index c9b9028f..f3115c68 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -11,6 +11,7 @@ """ import os +from datetime import timedelta from glob import glob try: @@ -361,6 +362,14 @@ CELERY_TASK_TIME_LIMIT = 30 * 60 CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0") CELERY_RESULT_BACKEND = "django-db" +CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" +CELERY_BEAT_SCHEDULE = { + "delete-old-media-files-every-48-hours": { + "task": "Logs.tasks.delete_old_media_files", + "schedule": timedelta(hours=48), + "kwargs": {"max_age_hours": 48}, + }, +} try: from TEKDB.local_settings import * # noqa: F403 From 6a9f6617e6e965d8159695d6b190f1fd74f6ae68 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 15:15:43 -0800 Subject: [PATCH 69/98] move delete_expired_chunks tasks to TEKDB --- TEKDB/Logs/__init__.py | 2 -- TEKDB/Logs/admin.py | 27 ---------------------- TEKDB/Logs/apps.py | 6 ----- TEKDB/Logs/migrations/0001_initial.py | 33 --------------------------- TEKDB/Logs/migrations/__init__.py | 0 TEKDB/Logs/models.py | 17 -------------- TEKDB/Logs/tests/__init__.py | 0 TEKDB/Logs/tests/test_forms.py | 0 TEKDB/Logs/tests/test_models.py | 0 TEKDB/Logs/tests/test_views.py | 0 TEKDB/Logs/views.py | 1 - TEKDB/TEKDB/admin.py | 5 ++++ TEKDB/TEKDB/settings.py | 5 ++-- TEKDB/{Logs => TEKDB}/tasks.py | 25 +------------------- 14 files changed, 8 insertions(+), 113 deletions(-) delete mode 100644 TEKDB/Logs/__init__.py delete mode 100644 TEKDB/Logs/admin.py delete mode 100644 TEKDB/Logs/apps.py delete mode 100644 TEKDB/Logs/migrations/0001_initial.py delete mode 100644 TEKDB/Logs/migrations/__init__.py delete mode 100644 TEKDB/Logs/models.py delete mode 100644 TEKDB/Logs/tests/__init__.py delete mode 100644 TEKDB/Logs/tests/test_forms.py delete mode 100644 TEKDB/Logs/tests/test_models.py delete mode 100644 TEKDB/Logs/tests/test_views.py delete mode 100644 TEKDB/Logs/views.py rename TEKDB/{Logs => TEKDB}/tasks.py (69%) diff --git a/TEKDB/Logs/__init__.py b/TEKDB/Logs/__init__.py deleted file mode 100644 index 97e929b4..00000000 --- a/TEKDB/Logs/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# TEKDB/__init__.py - diff --git a/TEKDB/Logs/admin.py b/TEKDB/Logs/admin.py deleted file mode 100644 index 995fda03..00000000 --- a/TEKDB/Logs/admin.py +++ /dev/null @@ -1,27 +0,0 @@ -from django.contrib import admin -from Logs.models import ExpiredChunkDeletionLog - - -class ExpiredChunkDeletionLogAdmin(admin.ModelAdmin): - readonly_fields = [ - "file_name", - "file_path", - "file_size", - "original_created_at", - "deleted_at", - "reason", - "success", - "error_message", - ] - - def has_add_permission(self, request): - return False - - def has_delete_permission(self, request, obj=None): - return False - - def has_change_permission(self, request, obj=None): - return False - - -admin.site.register(ExpiredChunkDeletionLog, ExpiredChunkDeletionLogAdmin) diff --git a/TEKDB/Logs/apps.py b/TEKDB/Logs/apps.py deleted file mode 100644 index 2ca2e123..00000000 --- a/TEKDB/Logs/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class LogsConfig(AppConfig): - name = "Logs" - verbose_name = "Logs" diff --git a/TEKDB/Logs/migrations/0001_initial.py b/TEKDB/Logs/migrations/0001_initial.py deleted file mode 100644 index 3ba9a6e7..00000000 --- a/TEKDB/Logs/migrations/0001_initial.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.29 on 2026-03-04 21:07 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='MediaFileDeletionLog', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('file_name', models.CharField(max_length=255)), - ('file_path', models.CharField(max_length=500)), - ('file_size', models.CharField(max_length=50, null=True)), - ('original_created_at', models.DateTimeField()), - ('deleted_at', models.DateTimeField(auto_now_add=True)), - ('reason', models.CharField(default='age_policy', max_length=100)), - ('success', models.BooleanField(default=True)), - ('error_message', models.TextField(blank=True)), - ], - options={ - 'verbose_name': 'Partial Upload Deletion Log', - 'verbose_name_plural': 'Partial Upload Deletion Logs', - 'indexes': [models.Index(fields=['deleted_at'], name='Logs_mediaf_deleted_17c511_idx')], - }, - ), - ] diff --git a/TEKDB/Logs/migrations/__init__.py b/TEKDB/Logs/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TEKDB/Logs/models.py b/TEKDB/Logs/models.py deleted file mode 100644 index b33a85fc..00000000 --- a/TEKDB/Logs/models.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.db import models - - -class ExpiredChunkDeletionLog(models.Model): - file_name = models.CharField(max_length=255) - file_path = models.CharField(max_length=500) - file_size = models.CharField(max_length=50, null=True) - original_created_at = models.DateTimeField() - deleted_at = models.DateTimeField(auto_now_add=True) - reason = models.CharField(max_length=100, default="age_policy") - success = models.BooleanField(default=True) - error_message = models.TextField(blank=True) - - class Meta: - indexes = [models.Index(fields=["deleted_at"])] - verbose_name = "Partial Upload Deletion Log" - verbose_name_plural = "Partial Upload Deletion Logs" diff --git a/TEKDB/Logs/tests/__init__.py b/TEKDB/Logs/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TEKDB/Logs/tests/test_forms.py b/TEKDB/Logs/tests/test_forms.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TEKDB/Logs/tests/test_models.py b/TEKDB/Logs/tests/test_models.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TEKDB/Logs/tests/test_views.py b/TEKDB/Logs/tests/test_views.py deleted file mode 100644 index e69de29b..00000000 diff --git a/TEKDB/Logs/views.py b/TEKDB/Logs/views.py deleted file mode 100644 index 60f00ef0..00000000 --- a/TEKDB/Logs/views.py +++ /dev/null @@ -1 +0,0 @@ -# Create your views here. diff --git a/TEKDB/TEKDB/admin.py b/TEKDB/TEKDB/admin.py index cc551409..64ee642b 100644 --- a/TEKDB/TEKDB/admin.py +++ b/TEKDB/TEKDB/admin.py @@ -8,6 +8,7 @@ from dal import autocomplete from mimetypes import guess_type from django.templatetags.static import static +from django_celery_results.admin import GroupResult # from moderation.admin import ModerationAdmin import nested_admin @@ -1770,3 +1771,7 @@ class UsersAdmin(UserAdmin): admin.site.register(LookupAuthorType) admin.site.register(LookupUserInfo) # admin.site.register(CurrentVersion) + +admin.site.unregister(GroupResult) +# admin.site.unregister(CrontabScheduleAdmin) +# admin.site.unregister(ClockedScheduleAdmin) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index f3115c68..72da5604 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -71,7 +71,6 @@ "Lookup", "Accounts", "Relationships", - "Logs", "reversion", "django.contrib.sites", "django_resumable_async_upload", @@ -364,8 +363,8 @@ CELERY_RESULT_BACKEND = "django-db" CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_BEAT_SCHEDULE = { - "delete-old-media-files-every-48-hours": { - "task": "Logs.tasks.delete_old_media_files", + "delete-expired-chunks-every-48-hours": { + "task": "TEKDB.tasks.delete_expired_chunks", "schedule": timedelta(hours=48), "kwargs": {"max_age_hours": 48}, }, diff --git a/TEKDB/Logs/tasks.py b/TEKDB/TEKDB/tasks.py similarity index 69% rename from TEKDB/Logs/tasks.py rename to TEKDB/TEKDB/tasks.py index 14594123..3f289791 100644 --- a/TEKDB/Logs/tasks.py +++ b/TEKDB/TEKDB/tasks.py @@ -17,10 +17,8 @@ def bytes_to_readable(num_bytes, suffix="B"): @shared_task(bind=True, max_retries=3, autoretry_for=(Exception,)) -def delete_old_media_files(self, max_age_hours=24): - +def delete_expired_chunks(self, max_age_hours=24): from django.conf import settings - from Logs.models import ExpiredChunkDeletionLog target_dir = os.path.join( settings.MEDIA_ROOT, settings.ADMIN_RESUMABLE_CHUNK_FOLDER @@ -55,31 +53,10 @@ def delete_old_media_files(self, max_age_hours=24): ) os.remove(file_path) - ExpiredChunkDeletionLog.objects.create( - file_name=filename, - file_path=file_path, - file_size=file_size, - original_created_at=timezone.datetime.fromtimestamp( - mtime, tz=timezone.utc - ), - reason="age_policy", - success=True, - ) deleted.append(file_path) except Exception as e: logger.error(f"Failed to delete {file_path}: {e}") - ExpiredChunkDeletionLog.objects.create( - file_name=filename, - file_path=file_path, - file_size=file_size, - original_created_at=timezone.datetime.fromtimestamp( - mtime, tz=timezone.utc - ), - reason="age_policy", - success=False, - error_message=str(e), - ) failed.append(file_path) logger.info( From f14f7a071543ad5d9852e040e417a826f2b06573 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 16:05:51 -0800 Subject: [PATCH 70/98] move env vars out of common.yaml --- docker/common.yaml | 13 +------------ docker/docker-compose.yaml | 11 +++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/common.yaml b/docker/common.yaml index 4ef577f5..71fd0dc7 100644 --- a/docker/common.yaml +++ b/docker/common.yaml @@ -23,22 +23,11 @@ services: context: ../TEKDB/ dockerfile: ../TEKDB/Dockerfile restart: unless-stopped - env_file: - .env.dev ports: - "8000:8000" - environment: - ALLOWED_HOSTS: ${ALLOWED_HOSTS} - DEBUG: ${DEBUG} - SQL_ENGINE: ${SQL_ENGINE} - SQL_HOST: ${SQL_HOST} - SQL_PORT: ${SQL_PORT} - SQL_DATABASE: ${SQL_DATABASE} - SQL_USER: ${SQL_USER} - SQL_PASSWORD: ${SQL_PASSWORD} - SECRET_KEY: ${SECRET_KEY} - CELERY_BROKER_URL: redis://redis:6379/0 + healthcheck: test: ["CMD-SHELL", "python manage.py check --deploy 2>/dev/null || python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:8000/\")' 2>/dev/null || exit 0"] interval: 15s diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index bcc6bade..23d3676b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -14,6 +14,17 @@ services: condition: service_healthy redis: condition: service_started + environment: + ALLOWED_HOSTS: ${ALLOWED_HOSTS} + DEBUG: ${DEBUG} + SQL_ENGINE: ${SQL_ENGINE} + SQL_HOST: ${SQL_HOST} + SQL_PORT: ${SQL_PORT} + SQL_DATABASE: ${SQL_DATABASE} + SQL_USER: ${SQL_USER} + SQL_PASSWORD: ${SQL_PASSWORD} + SECRET_KEY: ${SECRET_KEY} + CELERY_BROKER_URL: redis://redis:6379/0 volumes: - ../TEKDB:/usr/src/app redis: From 2c95eec1b5f85d9f2aa5340a11d7a28499b33077 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 5 Mar 2026 16:12:17 -0800 Subject: [PATCH 71/98] add redis, celery, and celery beat to docker-compose.prod.local --- docker/docker-compose.prod.local.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docker/docker-compose.prod.local.yaml b/docker/docker-compose.prod.local.yaml index 421e3d93..13899a77 100644 --- a/docker/docker-compose.prod.local.yaml +++ b/docker/docker-compose.prod.local.yaml @@ -12,6 +12,11 @@ services: env_file: - .env.dev ports: [] + depends_on: + db: + condition: service_healthy + redis: + condition: service_started volumes: - static_volume:/usr/src/app/static - media_volume:/usr/src/app/media @@ -27,6 +32,18 @@ services: volumes: - static_volume:/vol/static/static:ro - media_volume:/vol/static/media:ro + redis: + extends: + file: common.yaml + service: redis + celery: + extends: + file: common.yaml + service: celery + celery-beat: + extends: + file: common.yaml + service: celery-beat volumes: tekdb_db_data: From d2fefb8167519b45174de74c62ddedd759a48796 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 6 Mar 2026 09:58:12 -0800 Subject: [PATCH 72/98] remove debug task --- TEKDB/TEKDB/celery.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/TEKDB/TEKDB/celery.py b/TEKDB/TEKDB/celery.py index 0f1cbcc4..2c42a1a1 100644 --- a/TEKDB/TEKDB/celery.py +++ b/TEKDB/TEKDB/celery.py @@ -15,8 +15,3 @@ # Load task modules from all registered Django apps. app.autodiscover_tasks() - - -@app.task(bind=True, ignore_result=True) -def debug_task(self): - print(f"Request: {self.request!r}") From 354fa9438375e0faece9f600dca85ea31d01cb4f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 6 Mar 2026 11:40:51 -0800 Subject: [PATCH 73/98] wip: add tests for delete_expired_chunks --- TEKDB/TEKDB/tests/test_tasks.py | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 TEKDB/TEKDB/tests/test_tasks.py diff --git a/TEKDB/TEKDB/tests/test_tasks.py b/TEKDB/TEKDB/tests/test_tasks.py new file mode 100644 index 00000000..37203235 --- /dev/null +++ b/TEKDB/TEKDB/tests/test_tasks.py @@ -0,0 +1,44 @@ +import os +import time +from django.test import TestCase, override_settings + +from TEKDB.tasks import delete_expired_chunks + + +class DeleteOldMediaFilesTest(TestCase): + def test_missing_dir_returns_none(self): + with override_settings( + MEDIA_ROOT="/nonexistent", ADMIN_RESUMABLE_CHUNK_FOLDER="does_not_exist" + ): + result = delete_expired_chunks.run(max_age_hours=24) + self.assertIsNone(result) + + def test_deletes_old_files_and_skips_new(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp_path: + target = os.path.join(tmp_path, "chunks") + os.makedirs(target) + + old_file = os.path.join(target, "old.bin") + new_file = os.path.join(target, "new.bin") + with open(old_file, "wb") as f: + f.write(b"old") + with open(new_file, "wb") as f: + f.write(b"new") + + now = time.time() + os.utime(old_file, (now - 48 * 3600, now - 48 * 3600)) # 48 hours old + os.utime(new_file, (now - 1 * 3600, now - 1 * 3600)) # 1 hour old + + with override_settings( + MEDIA_ROOT=tmp_path, ADMIN_RESUMABLE_CHUNK_FOLDER="chunks" + ): + result = delete_expired_chunks.run(max_age_hours=24) + + self.assertIsInstance(result, dict) + self.assertEqual(result["deleted"], 1) + self.assertEqual(result["skipped"], 1) + self.assertEqual(result["failed"], 0) + self.assertFalse(os.path.exists(old_file)) + self.assertTrue(os.path.exists(new_file)) From bcdd5fbf0b9c6b5fd33aac1e558a8ee516138381 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 6 Mar 2026 11:48:59 -0800 Subject: [PATCH 74/98] rename tests --- TEKDB/TEKDB/tests/test_tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TEKDB/TEKDB/tests/test_tasks.py b/TEKDB/TEKDB/tests/test_tasks.py index 37203235..2d44710a 100644 --- a/TEKDB/TEKDB/tests/test_tasks.py +++ b/TEKDB/TEKDB/tests/test_tasks.py @@ -5,7 +5,7 @@ from TEKDB.tasks import delete_expired_chunks -class DeleteOldMediaFilesTest(TestCase): +class DeleteExpiredChunksTest(TestCase): def test_missing_dir_returns_none(self): with override_settings( MEDIA_ROOT="/nonexistent", ADMIN_RESUMABLE_CHUNK_FOLDER="does_not_exist" @@ -13,7 +13,7 @@ def test_missing_dir_returns_none(self): result = delete_expired_chunks.run(max_age_hours=24) self.assertIsNone(result) - def test_deletes_old_files_and_skips_new(self): + def test_deletes_expired_chunks_and_skips_new(self): import tempfile with tempfile.TemporaryDirectory() as tmp_path: From ed2c2b45ae741ceb8cad4a0bf434989f5ac66ce9 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 10 Mar 2026 10:25:05 -0700 Subject: [PATCH 75/98] add scripts and services for celery in vagrant --- TEKDB/scripts/provision_celery.sh | 32 +++++++++++++++++++++++++++++++ deployment/celery-beat.service | 18 +++++++++++++++++ deployment/celery-worker.service | 18 +++++++++++++++++ scripts/Linux/update.sh | 4 +++- 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 TEKDB/scripts/provision_celery.sh create mode 100644 deployment/celery-beat.service create mode 100644 deployment/celery-worker.service diff --git a/TEKDB/scripts/provision_celery.sh b/TEKDB/scripts/provision_celery.sh new file mode 100644 index 00000000..a15e246f --- /dev/null +++ b/TEKDB/scripts/provision_celery.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Ensure script is run as root +if [ "$EUID" -ne 0 ]; then + echo "Please run as root: sudo bash $0" + exit 1 +fi + +PROJECT_DIR=/usr/local/apps/TEKDB +APP_USER=vagrant + +# Install Redis +apt-get install -y redis-server +systemctl enable redis-server +systemctl start redis-server + +# Create log and pid directories +mkdir -p /var/log/celery /var/run/celery +chown $APP_USER:$APP_USER /var/log/celery /var/run/celery + +# Copy service files into systemd +cp $PROJECT_DIR/deployment/celery-worker.service /etc/systemd/system/ +cp $PROJECT_DIR/deployment/celery-beat.service /etc/systemd/system/ + +# Enable and start services +systemctl daemon-reload +systemctl enable celery-worker +systemctl enable celery-beat +systemctl start celery-worker +systemctl start celery-beat + +echo "Done. Check status with: systemctl status celery-worker celery-beat" \ No newline at end of file diff --git a/deployment/celery-beat.service b/deployment/celery-beat.service new file mode 100644 index 00000000..a363ee89 --- /dev/null +++ b/deployment/celery-beat.service @@ -0,0 +1,18 @@ +[Unit] +Description=TEKDB Celery Beat Service +After=network.target postgresql.service redis.service + +[Service] +Type=simple +User=vagrant +Group=vagrant +WorkingDirectory=/usr/local/apps/TEKDB/TEKDB +Environment="DJANGO_SETTINGS_MODULE=TEKDB.settings" +ExecStart=/usr/local/apps/env/bin/celery -A TEKDB beat \ + --loglevel=info \ + --logfile=/var/log/celery/tekdb-beat.log \ + --scheduler django_celery_beat.schedulers:DatabaseScheduler +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/deployment/celery-worker.service b/deployment/celery-worker.service new file mode 100644 index 00000000..f60c3e53 --- /dev/null +++ b/deployment/celery-worker.service @@ -0,0 +1,18 @@ +[Unit] +Description=TEKDB Celery Worker +After=network.target postgresql.service redis.service + +[Service] +Type=simple +User=vagrant +Group=vagrant +WorkingDirectory=/usr/local/apps/TEKDB/TEKDB +Environment="DJANGO_SETTINGS_MODULE=TEKDB.settings" +ExecStart=/usr/local/apps/env/bin/celery -A TEKDB worker \ + --loglevel=info \ + --logfile=/var/log/celery/tekdb-worker.log +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/scripts/Linux/update.sh b/scripts/Linux/update.sh index 2394aaf2..2b874343 100755 --- a/scripts/Linux/update.sh +++ b/scripts/Linux/update.sh @@ -40,4 +40,6 @@ $PYTHON $PROJ_ROOT/TEKDB/manage.py migrate $PYTHON $PROJ_ROOT/TEKDB/manage.py collectstatic --no-input sudo service uwsgi restart -sudo service nginx restart \ No newline at end of file +sudo service nginx restart +sudo service celery-worker restart +sudo service celery-beat restart \ No newline at end of file From 17a3ffb4e7597723ebb66ba1a26143b2fc43f6f2 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 10 Mar 2026 13:33:35 -0700 Subject: [PATCH 76/98] add CELERY_RESULT_EXTENDED setting --- TEKDB/TEKDB/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 65b90369..a7748a51 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -361,6 +361,7 @@ CELERY_TASK_TIME_LIMIT = 30 * 60 CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0") CELERY_RESULT_BACKEND = "django-db" +CELERY_RESULT_EXTENDED = True CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" CELERY_BEAT_SCHEDULE = { "delete-expired-chunks-every-48-hours": { From 4ffb47f8e52177ae0c8be54a2e1bc10934a7e756 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 10 Mar 2026 13:55:40 -0700 Subject: [PATCH 77/98] remove celery and celery beat restart commands from update.sh --- scripts/Linux/update.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/Linux/update.sh b/scripts/Linux/update.sh index 2b874343..2394aaf2 100755 --- a/scripts/Linux/update.sh +++ b/scripts/Linux/update.sh @@ -40,6 +40,4 @@ $PYTHON $PROJ_ROOT/TEKDB/manage.py migrate $PYTHON $PROJ_ROOT/TEKDB/manage.py collectstatic --no-input sudo service uwsgi restart -sudo service nginx restart -sudo service celery-worker restart -sudo service celery-beat restart \ No newline at end of file +sudo service nginx restart \ No newline at end of file From c24318bda8827cb33cc472aed2017e659d7160e0 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 10 Mar 2026 15:54:31 -0700 Subject: [PATCH 78/98] set redis service name as redis-server in celery service files --- TEKDB/TEKDB/settings.py | 2 +- deployment/celery-beat.service | 2 +- deployment/celery-worker.service | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index a7748a51..23d36844 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -359,7 +359,7 @@ CELERY_TIMEZONE = TIME_ZONE CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 30 * 60 -CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0") +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379/0") CELERY_RESULT_BACKEND = "django-db" CELERY_RESULT_EXTENDED = True CELERY_BEAT_SCHEDULER = "django_celery_beat.schedulers:DatabaseScheduler" diff --git a/deployment/celery-beat.service b/deployment/celery-beat.service index a363ee89..9935070c 100644 --- a/deployment/celery-beat.service +++ b/deployment/celery-beat.service @@ -1,6 +1,6 @@ [Unit] Description=TEKDB Celery Beat Service -After=network.target postgresql.service redis.service +After=network.target postgresql.service redis-server.service [Service] Type=simple diff --git a/deployment/celery-worker.service b/deployment/celery-worker.service index f60c3e53..7e9ee905 100644 --- a/deployment/celery-worker.service +++ b/deployment/celery-worker.service @@ -1,6 +1,6 @@ [Unit] Description=TEKDB Celery Worker -After=network.target postgresql.service redis.service +After=network.target postgresql.service redis-server.service [Service] Type=simple From bd06fda97c67452a7b23d45ef0d41a058038c98a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 10 Mar 2026 16:12:16 -0700 Subject: [PATCH 79/98] add redis, celery and celery-beat to docker-compose.prod.yaml; remove exit from healthcheck --- docker/common.yaml | 2 +- docker/docker-compose.prod.yaml | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/common.yaml b/docker/common.yaml index 71fd0dc7..7193a6ab 100644 --- a/docker/common.yaml +++ b/docker/common.yaml @@ -29,7 +29,7 @@ services: - "8000:8000" healthcheck: - test: ["CMD-SHELL", "python manage.py check --deploy 2>/dev/null || python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:8000/\")' 2>/dev/null || exit 0"] + test: ["CMD-SHELL", "python manage.py check --deploy 2>/dev/null || python -c 'import urllib.request; urllib.request.urlopen(\"http://localhost:8000/\")' 2>/dev/null"] interval: 15s timeout: 10s retries: 5 diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index 2105577a..bf2b60ce 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -13,6 +13,11 @@ services: env_file: - .env.prod ports: [] + depends_on: + db: + condition: service_healthy + redis: + condition: service_started volumes: - static_volume:/usr/src/app/static - media_volume:/usr/src/app/media @@ -26,6 +31,18 @@ services: volumes: - static_volume:/vol/static/static:ro - media_volume:/vol/static/media:ro + redis: + extends: + file: common.yaml + service: redis + celery: + extends: + file: common.yaml + service: celery + celery-beat: + extends: + file: common.yaml + service: celery-beat volumes: tekdb_db_data: From ad6f113b774c82a080690dbfbae6eaad3f40b33f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 11 Mar 2026 09:24:33 -0700 Subject: [PATCH 80/98] delete delete_expired_chunks.py --- TEKDB/TEKDB/management/commands/delete_expired_chunks.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 TEKDB/TEKDB/management/commands/delete_expired_chunks.py diff --git a/TEKDB/TEKDB/management/commands/delete_expired_chunks.py b/TEKDB/TEKDB/management/commands/delete_expired_chunks.py deleted file mode 100644 index e69de29b..00000000 From 9633ef598df4543470df7f422140bf9373a1e3b7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 11 Mar 2026 10:01:34 -0700 Subject: [PATCH 81/98] remove unnecessary env vars from celery and celery beat containers --- TEKDB/.env.dev | 1 + docker/common.yaml | 26 ++------------------------ 2 files changed, 3 insertions(+), 24 deletions(-) diff --git a/TEKDB/.env.dev b/TEKDB/.env.dev index e145b076..ab8cb1ff 100644 --- a/TEKDB/.env.dev +++ b/TEKDB/.env.dev @@ -7,3 +7,4 @@ SQL_USER=postgres SQL_PASSWORD=tekdb_password SQL_HOST=db SQL_PORT=5432 +CELERY_BROKER_URL=redis://redis:6379/0 \ No newline at end of file diff --git a/docker/common.yaml b/docker/common.yaml index 7193a6ab..83544190 100644 --- a/docker/common.yaml +++ b/docker/common.yaml @@ -41,19 +41,8 @@ services: command: celery -A TEKDB worker -l info volumes: - ../TEKDB:/usr/src/app - env_file: - - .env.dev environment: - ALLOWED_HOSTS: ${ALLOWED_HOSTS} - DEBUG: ${DEBUG} - SQL_ENGINE: ${SQL_ENGINE} - SQL_HOST: ${SQL_HOST} - SQL_PORT: ${SQL_PORT} - SQL_DATABASE: ${SQL_DATABASE} - SQL_USER: ${SQL_USER} - SQL_PASSWORD: ${SQL_PASSWORD} - SECRET_KEY: ${SECRET_KEY} - CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL} depends_on: web: condition: service_healthy @@ -67,19 +56,8 @@ services: command: celery -A TEKDB beat -l info volumes: - ../TEKDB:/usr/src/app - env_file: - - .env.dev environment: - ALLOWED_HOSTS: ${ALLOWED_HOSTS} - DEBUG: ${DEBUG} - SQL_ENGINE: ${SQL_ENGINE} - SQL_HOST: ${SQL_HOST} - SQL_PORT: ${SQL_PORT} - SQL_DATABASE: ${SQL_DATABASE} - SQL_USER: ${SQL_USER} - SQL_PASSWORD: ${SQL_PASSWORD} - SECRET_KEY: ${SECRET_KEY} - CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_BROKER_URL: ${CELERY_BROKER_URL} depends_on: web: condition: service_healthy From ce842411613776ce40fb8af7110d13d32a93a156 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 11 Mar 2026 10:28:45 -0700 Subject: [PATCH 82/98] run as non-root user in docker to resolve warning in celery worker --- TEKDB/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index 78fb2bd3..b9762e8e 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -34,6 +34,12 @@ COPY . /usr/src/app COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh +# Create a non-root user and give it ownership of the app directory +RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser \ + && chown -R appuser:appgroup /usr/src/app + +USER appuser + # Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000) EXPOSE 8000 From 4f1de260ebea8d815cc468fd3fa6bbb8fd1351cb Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 11 Mar 2026 11:19:50 -0700 Subject: [PATCH 83/98] add test for failed to delete chunk case --- TEKDB/TEKDB/tests/test_tasks.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/TEKDB/TEKDB/tests/test_tasks.py b/TEKDB/TEKDB/tests/test_tasks.py index 2d44710a..6cdb85d7 100644 --- a/TEKDB/TEKDB/tests/test_tasks.py +++ b/TEKDB/TEKDB/tests/test_tasks.py @@ -1,5 +1,6 @@ import os import time +from unittest.mock import patch from django.test import TestCase, override_settings from TEKDB.tasks import delete_expired_chunks @@ -42,3 +43,31 @@ def test_deletes_expired_chunks_and_skips_new(self): self.assertEqual(result["failed"], 0) self.assertFalse(os.path.exists(old_file)) self.assertTrue(os.path.exists(new_file)) + + def test_failed_to_delete_expired_chunks(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp_path: + target = os.path.join(tmp_path, "chunks") + os.makedirs(target) + + old_file = os.path.join(target, "old.bin") + with open(old_file, "wb") as f: + f.write(b"old") + + now = time.time() + os.utime(old_file, (now - 48 * 3600, now - 48 * 3600)) # 48 hours old + + with override_settings( + MEDIA_ROOT=tmp_path, ADMIN_RESUMABLE_CHUNK_FOLDER="chunks" + ): + with patch( + "os.remove", side_effect=PermissionError("permission denied") + ): + result = delete_expired_chunks.run(max_age_hours=24) + + self.assertIsInstance(result, dict) + self.assertEqual(result["deleted"], 0) + self.assertEqual(result["skipped"], 0) + self.assertEqual(result["failed"], 1) + self.assertTrue(os.path.exists(old_file)) From 3d2bc99f86fa8799a99e1378c98d321f2147c70e Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 12 Mar 2026 12:23:31 -0700 Subject: [PATCH 84/98] add celery_broker_url tf variable --- infra/ec2.tf | 1 + infra/main.tf | 5 +++++ infra/user_data.tftpl | 5 +++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/infra/ec2.tf b/infra/ec2.tf index 70e066b9..6fb1c5e7 100644 --- a/infra/ec2.tf +++ b/infra/ec2.tf @@ -28,6 +28,7 @@ resource "aws_instance" "itkdb" { sql_db_password = var.sql_db_password sql_port = var.sql_port django_allowed_hosts = var.django_allowed_hosts + celery_broker_url = var.celery_broker_url }) root_block_device { diff --git a/infra/main.tf b/infra/main.tf index e992b052..7fb7a059 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -91,4 +91,9 @@ variable "web_ecr_image_uri" { variable "proxy_ecr_image_uri" { description = "ECR image URI for the proxy" type = string +} + +variable "celery_broker_url" { + description = "Broker URL for Celery" + type = string } \ No newline at end of file diff --git a/infra/user_data.tftpl b/infra/user_data.tftpl index ed2b71e6..b42ede35 100644 --- a/infra/user_data.tftpl +++ b/infra/user_data.tftpl @@ -48,10 +48,10 @@ git clone https://github.com/Ecotrust/TEKDB.git cd TEKDB/ # TODO: remove once the branch is merged to main -git checkout docker-nginx +git checkout develop # pull the latest changes from the remote repository -git pull origin docker-nginx +git pull origin develop # copy environment variables into the docker directory echo "Creating .env.prod file..." @@ -68,6 +68,7 @@ SQL_HOST=${sql_host} SQL_PORT=${sql_port} ITKDB_ECR_PATH=${web_ecr_image_uri} ITKDB_PROXY_ECR_PATH=${proxy_ecr_image_uri} +CELERY_BROKER_URL=${celery_broker_url} EOF echo "Logging in to AWS ECR for web image..." From 7232b70ada4b3efccc9cf4b8eaacba82c6dd9ebc Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Thu, 12 Mar 2026 13:13:09 -0700 Subject: [PATCH 85/98] remove non root user from dockerfile --- TEKDB/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/TEKDB/Dockerfile b/TEKDB/Dockerfile index b9762e8e..535b6cce 100644 --- a/TEKDB/Dockerfile +++ b/TEKDB/Dockerfile @@ -35,10 +35,12 @@ COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh # Create a non-root user and give it ownership of the app directory -RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser \ - && chown -R appuser:appgroup /usr/src/app +# TODO: consider using a non-root user for better security. Addings just this +# resulted in a permissions error, "ERROR: Unable to load local_settings.py." +# RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser \ +# && chown -R appuser:appgroup /usr/src/app -USER appuser +# USER appuser # Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000) EXPOSE 8000 From 719afe003f44762c3647313c839227ef387d29bb Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 11:03:53 -0700 Subject: [PATCH 86/98] add env file to celery and celery-beat in prod docker-compose --- docker/docker-compose.prod.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index bf2b60ce..9c3610ac 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -39,10 +39,14 @@ services: extends: file: common.yaml service: celery + env_file: + - .env.prod celery-beat: extends: file: common.yaml service: celery-beat + env_file: + - .env.prod volumes: tekdb_db_data: From d9143c0c2765409e55d0a4b3e21c504451a9ebf6 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 11:38:14 -0700 Subject: [PATCH 87/98] add logs to user_data.tfpl; move code to tekdb dir --- infra/user_data.tftpl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/infra/user_data.tftpl b/infra/user_data.tftpl index b42ede35..11bb627d 100644 --- a/infra/user_data.tftpl +++ b/infra/user_data.tftpl @@ -40,17 +40,24 @@ sudo usermod -aG docker ubuntu echo "Installing git..." sudo apt install git -y +echo "Making tekdb directory..." mkdir tekdb + +echo "Changing to tekdb directory..." +cd tekdb # clone the TEKDB repo +echo "Cloning TEKDB repository..." git clone https://github.com/Ecotrust/TEKDB.git +echo "Changing to TEKDB repository..." cd TEKDB/ -# TODO: remove once the branch is merged to main +echo "Checking out develop branch..." git checkout develop # pull the latest changes from the remote repository +echo "Pulling latest changes from develop branch..." git pull origin develop # copy environment variables into the docker directory From 2ddd2d27f976fe615beb47c9dfc82537ccbd67c7 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 11:38:39 -0700 Subject: [PATCH 88/98] add caching to deploy ec2 gh action --- .github/workflows/deploy-ec2-staging.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 55e4eeb6..e62a1ead 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -38,6 +38,8 @@ jobs: run: | docker buildx build \ --platform linux/amd64,linux/arm64 \ + --cache-from type=gha,scope=web \ + --cache-to type=gha,mode=max,scope=web \ -t $IMAGE_URI:$IMAGE_TAG \ -t $IMAGE_URI:latest \ -f ./TEKDB/Dockerfile ./TEKDB \ @@ -51,6 +53,8 @@ jobs: run: | docker buildx build \ --platform linux/amd64,linux/arm64 \ + --cache-from type=gha,scope=proxy \ + --cache-to type=gha,mode=max,scope=proxy \ -t $IMAGE_URI:$IMAGE_TAG \ -t $IMAGE_URI:latest \ -f ./proxy/Dockerfile ./proxy \ From d539283705b93322fe134c3cce04c74e797abda1 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 14:15:33 -0700 Subject: [PATCH 89/98] update logic in entrypoint to load fixtures --- TEKDB/TEKDB/settings.py | 5 ++++- TEKDB/entrypoint.sh | 6 ++++-- infra/user_data.tftpl | 4 ++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index 23d36844..f093fdd0 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -374,6 +374,9 @@ try: from TEKDB.local_settings import * # noqa: F403 except Exception: + import sys + print( - "ERROR: Unable to load local_settings.py. This is expected for docker deployment" + "ERROR: Unable to load local_settings.py. This is expected for docker deployment", + file=sys.stderr, ) diff --git a/TEKDB/entrypoint.sh b/TEKDB/entrypoint.sh index a0644cd0..9bfa8df0 100644 --- a/TEKDB/entrypoint.sh +++ b/TEKDB/entrypoint.sh @@ -22,12 +22,14 @@ echo "Applying database migrations..." python manage.py migrate --noinput # Load default users only if no users exist echo "Checking for existing users..." -if [ "$(python manage.py shell -c 'from django.contrib.auth import get_user_model; print(get_user_model().objects.count())')" = "0" ]; then +if [ "$(python manage.py shell -c 'from django.contrib.auth import get_user_model; print(get_user_model().objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No users found, loading default users fixture..." python manage.py loaddata TEKDB/fixtures/default_users_fixture.json fi # Load default lookups only if no lookups exist. Use LookupPlanningUnit as the check. echo "Checking for existing lookups..." -if [ "$(python manage.py shell -c 'from TEKDB.models import LookupPlanningUnit; print(LookupPlanningUnit.objects.count())')" = "0" ]; then +if [ "$(python manage.py shell -c 'from TEKDB.models import LookupPlanningUnit; print(LookupPlanningUnit.objects.count())' 2>/dev/null | tail -1)" = "0" ]; then + echo "No lookups found, loading default lookups fixture..." python manage.py loaddata TEKDB/fixtures/default_lookups_fixture.json fi diff --git a/infra/user_data.tftpl b/infra/user_data.tftpl index 11bb627d..7220d7ac 100644 --- a/infra/user_data.tftpl +++ b/infra/user_data.tftpl @@ -62,7 +62,7 @@ git pull origin develop # copy environment variables into the docker directory echo "Creating .env.prod file..." -cat > /TEKDB/docker/.env.prod < docker/.env.prod < Date: Fri, 13 Mar 2026 15:17:52 -0700 Subject: [PATCH 90/98] unlimited client_max_body_size --- proxy/default.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/proxy/default.conf b/proxy/default.conf index 8ef2d0a5..4ac0069a 100644 --- a/proxy/default.conf +++ b/proxy/default.conf @@ -1,5 +1,6 @@ server { listen 8080; + client_max_body_size 0; location /static { alias /vol/static/static; From 9b2626b81fe1e34bb1da5be1b710951bc66b2936 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 15:55:15 -0700 Subject: [PATCH 91/98] fix location for tekdb repo in gh action --- .github/workflows/deploy-ec2-staging.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index e62a1ead..2fb7c6b9 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -79,6 +79,7 @@ jobs: "sudo docker pull ${{ secrets.WEB_ECR_IMAGE_URI }}:latest", "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.PROXY_ECR_IMAGE_URI }}", "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", + "cd /tekdb", "cd /TEKDB", "sudo docker compose down", "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d", From d1d6c6409ea910a93bc2dca8399d7a8dd8b7bbff Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Fri, 13 Mar 2026 16:36:14 -0700 Subject: [PATCH 92/98] use absolute paths in ssm command --- .github/workflows/deploy-ec2-staging.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 2fb7c6b9..925544b6 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -79,10 +79,8 @@ jobs: "sudo docker pull ${{ secrets.WEB_ECR_IMAGE_URI }}:latest", "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.PROXY_ECR_IMAGE_URI }}", "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest", - "cd /tekdb", - "cd /TEKDB", - "sudo docker compose down", - "docker compose --env-file docker/.env.prod -f docker/docker-compose.prod.yaml up -d", + "sudo docker compose -f /tekdb/TEKDB/docker/docker-compose.prod.yaml down", + "docker compose --env-file /tekdb/TEKDB/docker/.env.prod -f /tekdb/TEKDB/docker/docker-compose.prod.yaml up -d", "sleep 5", "docker ps --filter name=web --format \"{{.Status}}\"" ]' \ From 54921deef82aeb844e13ef9dd8b4d361b2542423 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 16 Mar 2026 13:09:50 -0700 Subject: [PATCH 93/98] add settings for logging --- TEKDB/TEKDB/settings.py | 38 ++++++++++++++++++++++++++++++++++++++ TEKDB/TEKDB/tasks.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index f093fdd0..ec71391c 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -371,6 +371,44 @@ }, } +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "verbose": { + "format": "{name} {levelname} {asctime} {module} {process:d} {thread:d} {message}", + "style": "{", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "verbose", + }, + }, + "root": { + "handlers": ["console"], + "level": "WARNING", + }, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "celery.task": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "delete_expired_chunks": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, +} + try: from TEKDB.local_settings import * # noqa: F403 except Exception: diff --git a/TEKDB/TEKDB/tasks.py b/TEKDB/TEKDB/tasks.py index 3f289791..88efad31 100644 --- a/TEKDB/TEKDB/tasks.py +++ b/TEKDB/TEKDB/tasks.py @@ -5,7 +5,7 @@ from django.utils import timezone -logger = logging.getLogger("delete_partial_upload") +logger = logging.getLogger("delete_expired_chunks") def bytes_to_readable(num_bytes, suffix="B"): From 76768b1b1daf6a7662aa673846a274648384127f Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 16 Mar 2026 14:02:25 -0700 Subject: [PATCH 94/98] bind media volumes to celery and celery-beat; use ECR images in prod celery and celery beat --- docker/docker-compose.prod.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/docker-compose.prod.yaml b/docker/docker-compose.prod.yaml index 9c3610ac..e7f2bd41 100644 --- a/docker/docker-compose.prod.yaml +++ b/docker/docker-compose.prod.yaml @@ -39,14 +39,20 @@ services: extends: file: common.yaml service: celery + image: ${ITKDB_ECR_PATH}:latest env_file: - .env.prod + volumes: + - media_volume:/usr/src/app/media celery-beat: extends: file: common.yaml service: celery-beat + image: ${ITKDB_ECR_PATH}:latest env_file: - .env.prod + volumes: + - media_volume:/usr/src/app/media volumes: tekdb_db_data: From 2ce08bbde6ff0533deb4c2351d61f8bb879f637e Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Mon, 16 Mar 2026 15:13:07 -0700 Subject: [PATCH 95/98] dont inherit build and volumes for celery and celery-beat git pull in deploy ec2 gh action --- .github/workflows/deploy-ec2-staging.yml | 1 + docker/common.yaml | 6 ------ docker/docker-compose.prod.local.yaml | 6 ++++++ docker/docker-compose.yaml | 6 ++++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-ec2-staging.yml b/.github/workflows/deploy-ec2-staging.yml index 925544b6..d26a4fb1 100644 --- a/.github/workflows/deploy-ec2-staging.yml +++ b/.github/workflows/deploy-ec2-staging.yml @@ -75,6 +75,7 @@ jobs: --instance-ids "${{ steps.get-instance.outputs.instance_id }}" \ --document-name "AWS-RunShellScript" \ --parameters 'commands=[ + "cd /tekdb/TEKDB && git pull origin develop", "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.WEB_ECR_IMAGE_URI }}", "sudo docker pull ${{ secrets.WEB_ECR_IMAGE_URI }}:latest", "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.PROXY_ECR_IMAGE_URI }}", diff --git a/docker/common.yaml b/docker/common.yaml index 83544190..8db4b675 100644 --- a/docker/common.yaml +++ b/docker/common.yaml @@ -36,11 +36,8 @@ services: start_period: 30s celery: - build: ../TEKDB/ entrypoint: [] command: celery -A TEKDB worker -l info - volumes: - - ../TEKDB:/usr/src/app environment: CELERY_BROKER_URL: ${CELERY_BROKER_URL} depends_on: @@ -51,11 +48,8 @@ services: redis: condition: service_started celery-beat: - build: ../TEKDB/ entrypoint: [] command: celery -A TEKDB beat -l info - volumes: - - ../TEKDB:/usr/src/app environment: CELERY_BROKER_URL: ${CELERY_BROKER_URL} depends_on: diff --git a/docker/docker-compose.prod.local.yaml b/docker/docker-compose.prod.local.yaml index 13899a77..358fb020 100644 --- a/docker/docker-compose.prod.local.yaml +++ b/docker/docker-compose.prod.local.yaml @@ -37,10 +37,16 @@ services: file: common.yaml service: redis celery: + build: ../TEKDB/ + volumes: + - ../TEKDB:/usr/src/app extends: file: common.yaml service: celery celery-beat: + build: ../TEKDB/ + volumes: + - ../TEKDB:/usr/src/app extends: file: common.yaml service: celery-beat diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 23d3676b..d3adb7f7 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -32,10 +32,16 @@ services: file: common.yaml service: redis celery: + build: ../TEKDB/ + volumes: + - ../TEKDB:/usr/src/app extends: file: common.yaml service: celery celery-beat: + build: ../TEKDB/ + volumes: + - ../TEKDB:/usr/src/app extends: file: common.yaml service: celery-beat From 29c9341b5b19aa8ccfdc3ace71f240f496f9da5c Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Tue, 17 Mar 2026 14:44:49 -0700 Subject: [PATCH 96/98] clear ContentType cache in import database --- TEKDB/TEKDB/views.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/TEKDB/TEKDB/views.py b/TEKDB/TEKDB/views.py index a342c645..6377242c 100644 --- a/TEKDB/TEKDB/views.py +++ b/TEKDB/TEKDB/views.py @@ -3,6 +3,7 @@ from datetime import datetime from django.conf import settings from django.contrib.auth.decorators import user_passes_test, permission_required +from django.contrib.contenttypes.models import ContentType from django.core import management from django.db import connection from django.db.models import Q @@ -222,6 +223,13 @@ def ImportDatabase(request): fixture_file_path = target_fixture.name management.call_command("loaddata", fixture_file_path) + # Clear Django's in-memory ContentType cache so subsequent + # requests use the PKs from the newly imported data, not + # the stale pre-import values. Without this, the + # ResumableAdminWidget renders the wrong content_type_id + # (e.g. the old Media PK now maps to LookupTechniques), + # causing FieldDoesNotExist on upload. + ContentType.objects.clear_cache() except Exception as e: status_code = 500 status_message = "Error while loading in data from provided zipfile. Your old data has been removed. Please coordinate with IT to restore your database, and share this error message with them:\n {}".format( From 8b1c601c7f662879b862d38e783164895bed9654 Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 18 Mar 2026 11:35:02 -0700 Subject: [PATCH 97/98] add test assertion for clear_cache run --- TEKDB/TEKDB/tests/test_views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/tests/test_views.py b/TEKDB/TEKDB/tests/test_views.py index 47fbf429..e7642b1c 100644 --- a/TEKDB/TEKDB/tests/test_views.py +++ b/TEKDB/TEKDB/tests/test_views.py @@ -520,7 +520,9 @@ def test_successful_import(self): ) self.import_request.FILES["import_file"] = import_file self.import_request.user = Users.objects.get(username="admin") - ImportDatabase(self.import_request) + with patch("TEKDB.views.ContentType.objects.clear_cache") as clear_cache: + ImportDatabase(self.import_request) + clear_cache.assert_called_once_with() self.assertEqual(Resources.objects.all().count(), self.old_resources_count) self.assertEqual( Resources.objects.filter(commonname=self.dummy_1_name).count(), 0 From 30986013f43260647f136c9d475767ce6da6de4a Mon Sep 17 00:00:00 2001 From: Paige Williams Date: Wed, 18 Mar 2026 14:51:01 -0700 Subject: [PATCH 98/98] bump version to 2.9.0 --- TEKDB/TEKDB/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TEKDB/TEKDB/settings.py b/TEKDB/TEKDB/settings.py index ec71391c..f43f8559 100644 --- a/TEKDB/TEKDB/settings.py +++ b/TEKDB/TEKDB/settings.py @@ -262,7 +262,7 @@ TINYMCE_FILEBROWSER = False # Add Version to the admin site header -VERSION = "2.8.0" +VERSION = "2.9.0" ADMIN_SITE_HEADER = os.environ.get( "ADMIN_SITE_HEADER", default="ITK DB Admin v{}".format(VERSION) )