Interruptible uploads + delete chunks in celery task

.github/workflows/create-and-publish-docker-images.yml

@@ -0,0 +1,80 @@
+# source: https://docs.github.com/en/actions/tutorials/publish-packages/publish-docker-images#publishing-images-to-docker-hub-and-github-packages
+name: Create and publish a Docker image
+
+# manually trigger while testing 
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `PACKAGE_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.PACKAGE_TOKEN }}
+      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
+      - name: Set lowercase image name
+        run: |
+          IMAGE_NAME=$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')
+          echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
+      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see [Usage](https://github.com/docker/build-push-action#usage) in the README of the `docker/build-push-action` repository.
+      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+
+      - name: Build and push web Docker image
+        id: push
+        uses: docker/build-push-action@v6
+        with:
+          context: ./TEKDB
+          file: ./TEKDB/Dockerfile
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:${{ github.sha }}
+          labels: ${{ steps.meta.outputs.labels }}
+      - name: Build and push proxy Docker image
+        id: push-proxy
+        uses: docker/build-push-action@v6
+        with:
+          context: ./proxy
+          file: ./proxy/Dockerfile
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy:latest,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy:${{ github.sha }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see [Using artifact attestations to establish provenance for builds](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds).
+      - name: Generate artifact attestation for web image
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true
+      - name: Generate artifact attestation for proxy image
+        uses: actions/attest-build-provenance@v3
+        with:
+          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/proxy
+          subject-digest: ${{ steps.push-proxy.outputs.digest }}
+          push-to-registry: true
+

.github/workflows/deploy-ec2-staging.yml

@@ -0,0 +1,111 @@
+name: Build, Push, and Deploy to EC2 Staging
+
+on:
+  push:
+    branches:
+      - develop
+  workflow_dispatch:
+
+jobs:
+  build-push-and-deploy:
+    name: Build, Push to ECR, Deploy to EC2 Staging
+    runs-on: ubuntu-latest
+    permissions: 
+        id-token: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v5
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
+          aws-region: ${{ secrets.AWS_REGION}}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build, tag, and push web Docker image to ECR
+        id: build-web-image
+        env:
+          IMAGE_TAG: ${{ github.sha }}
+          IMAGE_URI: ${{ secrets.WEB_ECR_IMAGE_URI }}
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --cache-from type=gha,scope=web \
+            --cache-to type=gha,mode=max,scope=web \
+            -t $IMAGE_URI:$IMAGE_TAG \
+            -t $IMAGE_URI:latest \
+            -f ./TEKDB/Dockerfile ./TEKDB \
+            --push
+      
+      - name: Build, tag, and push proxy Docker image to ECR
+        id: build-proxy-image
+        env:
+          IMAGE_TAG: ${{ github.sha }}
+          IMAGE_URI: ${{ secrets.PROXY_ECR_IMAGE_URI }}
+        run: |
+          docker buildx build \
+            --platform linux/amd64,linux/arm64 \
+            --cache-from type=gha,scope=proxy \
+            --cache-to type=gha,mode=max,scope=proxy \
+            -t $IMAGE_URI:$IMAGE_TAG \
+            -t $IMAGE_URI:latest \
+            -f ./proxy/Dockerfile ./proxy \
+            --push
+      - name: get EC2 Instance ID
+        id: get-instance
+        run: |
+          INSTANCE_ID=$(aws ec2 describe-instances \
+            --filters "Name=tag:Name,Values=${{ secrets.EC2_INSTANCE_NAME }}" \
+                      "Name=instance-state-name,Values=running" \
+            --query "Reservations[0].Instances[0].InstanceId" \
+            --output text)
+          echo "instance_id=$INSTANCE_ID" >> $GITHUB_OUTPUT
+      - name: Deploy via SSM
+        id: ssm-deploy
+        run: |
+          COMMAND_ID=$(aws ssm send-command \
+            --instance-ids "${{ steps.get-instance.outputs.instance_id }}" \
+            --document-name "AWS-RunShellScript" \
+            --parameters 'commands=[
+              "cd /tekdb/TEKDB && git pull origin develop",
+              "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.WEB_ECR_IMAGE_URI }}",
+              "sudo docker pull ${{ secrets.WEB_ECR_IMAGE_URI }}:latest",
+              "aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | sudo docker login --username AWS --password-stdin ${{ secrets.PROXY_ECR_IMAGE_URI }}",
+              "sudo docker pull ${{ secrets.PROXY_ECR_IMAGE_URI }}:latest",
+              "sudo docker compose -f /tekdb/TEKDB/docker/docker-compose.prod.yaml down",
+              "docker compose --env-file /tekdb/TEKDB/docker/.env.prod -f /tekdb/TEKDB/docker/docker-compose.prod.yaml up -d",
+              "sleep 5",
+              "docker ps --filter name=web --format \"{{.Status}}\""
+            ]' \
+            --timeout-seconds 120 \
+            --comment "Deploy ${{ github.sha }}" \
+            --query "Command.CommandId" \
+            --output text)
+          echo "command_id=$COMMAND_ID" >> $GITHUB_OUTPUT
+      - name: Wait for deployment to complete
+        run: |
+          aws ssm wait command-executed \
+            --command-id ${{ steps.ssm-deploy.outputs.command_id }} \
+            --instance-id ${{ steps.get-instance.outputs.instance_id }}
+      - name: Verify deployment result
+        run: |
+          RESULT=$(aws ssm get-command-invocation \
+            --command-id ${{ steps.ssm-deploy.outputs.command_id }} \
+            --instance-id ${{ steps.get-instance.outputs.instance_id }} \
+            --query "{Status:Status,Output:StandardOutputContent,Error:StandardErrorContent}" \
+            --output json)
+          echo "$RESULT" | jq .
+          STATUS=$(echo "$RESULT" | jq -r '.Status')
+          if [ "$STATUS" != "Success" ]; then
+            echo "::error::Deployment failed with status: $STATUS"
+            exit 1
+          fi
+          echo "Deployment succeeded."

.gitignore

@@ -86,6 +86,7 @@ celerybeat-schedule
 
 # dotenv
 .env
+.env.prod
 
 # virtualenv
 venv/
@@ -98,4 +99,10 @@ ENV/
 .ropeproject
 
 #vscode settings
-.vscode/
+.vscode/
+
+# OpenTofu/Terraform
+.terraform/
+terraform.tfstate
+terraform.tfstate.backup
+*.tfvars

TEKDB/.env.dev

@@ -7,3 +7,4 @@ SQL_USER=postgres
 SQL_PASSWORD=tekdb_password
 SQL_HOST=db
 SQL_PORT=5432
+CELERY_BROKER_URL=redis://redis:6379/0

TEKDB/Dockerfile

@@ -22,6 +22,7 @@ WORKDIR /usr/src/app
 COPY requirements.txt requirements_linux.txt /usr/src/app/
 
 # Upgrade pip and install Python dependencies
+# Note: editable packages (-e) will be installed at runtime via entrypoint.sh
 RUN pip install --upgrade pip \
     && pip install -r requirements.txt -r requirements_linux.txt
 
@@ -33,6 +34,14 @@ COPY . /usr/src/app
 COPY entrypoint.sh /usr/local/bin/entrypoint.sh
 RUN chmod +x /usr/local/bin/entrypoint.sh
 
+# Create a non-root user and give it ownership of the app directory
+# TODO: consider using a non-root user for better security. Addings just this 
+# resulted in a permissions error, "ERROR: Unable to load local_settings.py."
+# RUN addgroup --system appgroup && adduser --system --ingroup appgroup appuser \
+#     && chown -R appuser:appgroup /usr/src/app
+
+# USER appuser
+
 # Expose the port the app runs on (entrypoint starts django development server or uWSGI on 8000)
 EXPOSE 8000
 
@@ -41,5 +50,3 @@ ENV DJANGO_SETTINGS_MODULE=TEKDB.settings
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 
-# use development server by default 
-CMD ["dev"]

TEKDB/TEKDB/__init__.py

@@ -1,2 +1,6 @@
 # TEKDB/__init__.py
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
 
+__all__ = ("celery_app",)

TEKDB/TEKDB/admin.py

@@ -8,6 +8,7 @@
 from dal import autocomplete
 from mimetypes import guess_type
 from django.templatetags.static import static
+from django_celery_results.admin import GroupResult
 
 # from moderation.admin import ModerationAdmin
 import nested_admin
@@ -61,6 +62,7 @@
 from TEKDB.settings import BASE_DIR
 from TEKDB.widgets import OpenLayers6Widget
 
+
 admin.site.site_header = ADMIN_SITE_HEADER
 
 
@@ -626,57 +628,69 @@ def save_model(self, request, obj, form, change):
         activities = form.cleaned_data.get("activities")
         placesresources = form.cleaned_data.get("placesresources")
 
-        for file in request.FILES.getlist("files"):
-            mime_type, _ = guess_type(file.name)
-            # if mime_type:
-            file_mime_type = mime_type.split("/")[0]
-            media_type_instance = LookupMediaType.objects.filter(
-                mediatype__startswith=file_mime_type
-            ).first()
-            if media_type_instance:
-                mediatype = media_type_instance
-            else:
-                media_type_instance = LookupMediaType.objects.filter(
-                    mediatype__startswith="other"
-                ).first()
+        # Handle async uploaded file (comes as a list of file path strings)
+        uploaded_file_paths = form.cleaned_data.get("files")
+
+        if uploaded_file_paths and isinstance(uploaded_file_paths, list):
+            import os
+
+            for uploaded_file_path in uploaded_file_paths:
+                # Extract just the filename from the path
+                file_name = os.path.basename(uploaded_file_path)
+
+                # Guess MIME type from filename
+                mime_type, _ = guess_type(file_name)
+                if mime_type:
+                    file_mime_type = mime_type.split("/")[0]
+                    media_type_instance = LookupMediaType.objects.filter(
+                        mediatype__startswith=file_mime_type
+                    ).first()
+                else:
+                    media_type_instance = None
+
+                if not media_type_instance:
+                    media_type_instance = LookupMediaType.objects.filter(
+                        mediatype__startswith="other"
+                    ).first()
+
                 mediatype = media_type_instance
-            filename = file.name.split(".")[0]
+                filename = file_name.rsplit(".", 1)[0]  # Remove extension
 
-            media_instance = Media(
-                medianame=filename,
-                mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}',
-                mediafile=file,
-                mediatype=mediatype,
-            )
-            media_instance.save()
-            obj.mediabulkupload.add(media_instance)
-
-            # Add relationships
-            if places:
-                for place in places:
-                    PlacesMediaEvents.objects.create(
-                        placeid=place, mediaid=media_instance
-                    )
-            if resources:
-                for resource in resources:
-                    ResourcesMediaEvents.objects.create(
-                        resourceid=resource, mediaid=media_instance
-                    )
-            if citations:
-                for citation in citations:
-                    MediaCitationEvents.objects.create(
-                        citationid=citation, mediaid=media_instance
-                    )
-            if activities:
-                for activity in activities:
-                    ResourceActivityMediaEvents.objects.create(
-                        resourceactivityid=activity, mediaid=media_instance
-                    )
-            if placesresources:
-                for placeresource in placesresources:
-                    PlacesResourceMediaEvents.objects.create(
-                        placeresourceid=placeresource, mediaid=media_instance
-                    )
+                media_instance = Media(
+                    medianame=filename,
+                    mediadescription=f'Part of the "{obj.mediabulkname}" Media Bulk Upload that was uploaded on {obj.mediabulkdate}',
+                    mediafile=uploaded_file_path,
+                    mediatype=mediatype,
+                )
+                media_instance.save()
+                obj.mediabulkupload.add(media_instance)
+
+                # Add relationships
+                if places:
+                    for place in places:
+                        PlacesMediaEvents.objects.create(
+                            placeid=place, mediaid=media_instance
+                        )
+                if resources:
+                    for resource in resources:
+                        ResourcesMediaEvents.objects.create(
+                            resourceid=resource, mediaid=media_instance
+                        )
+                if citations:
+                    for citation in citations:
+                        MediaCitationEvents.objects.create(
+                            citationid=citation, mediaid=media_instance
+                        )
+                if activities:
+                    for activity in activities:
+                        ResourceActivityMediaEvents.objects.create(
+                            resourceactivityid=activity, mediaid=media_instance
+                        )
+                if placesresources:
+                    for placeresource in placesresources:
+                        PlacesResourceMediaEvents.objects.create(
+                            placeresourceid=placeresource, mediaid=media_instance
+                        )
 
     @admin.display(description="Thumbnails")
     def thumbnail_gallery(self, obj):
@@ -1757,3 +1771,7 @@ class UsersAdmin(UserAdmin):
 admin.site.register(LookupAuthorType)
 admin.site.register(LookupUserInfo)
 # admin.site.register(CurrentVersion)
+
+admin.site.unregister(GroupResult)
+# admin.site.unregister(CrontabScheduleAdmin)
+# admin.site.unregister(ClockedScheduleAdmin)