Compute Worker Use docker-py instead of the current subprocess way of doing things (for podman and docker)

.github/workflows/build_cw_image-branch.yml

@@ -0,0 +1,27 @@
+name: build_CW_docker_image_branch
+on:
+  push:
+    # On push for every branch except develop and master since they have their own workflows
+    branches:
+      - '*'
+      - '!develop'
+      - '!master'
+    paths:
+      - Dockerfile.compute_worker
+      - compute_worker/**
+jobs:
+  build_push_image:
+    name: Build Docker Image then Push it to Docker.io
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v5
+      - name: Build Image
+        run: docker build -t codalab/competitions-v2-compute-worker:${{ github.ref_name }} -f Dockerfile.compute_worker .
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push image with branch name as tag
+        run: docker push codalab/competitions-v2-compute-worker:${{ github.ref_name }}

.github/workflows/build_cw_image-dev.yml

@@ -0,0 +1,24 @@
+name: build_CW_docker_image_develop
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - Dockerfile.compute_worker
+      - compute_worker/**
+jobs:
+  build:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v5
+      - name: Build Image
+        run: docker build -t codalab/competitions-v2-compute-worker:test -f Dockerfile.compute_worker .
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push image with test tag
+        run: docker push codalab/competitions-v2-compute-worker:test

.github/workflows/build_cw_image-prod.yml

@@ -0,0 +1,28 @@
+name: build_CW_docker_image_branch
+on:
+  push:
+    tags:
+      - '*'
+    paths:
+      - Dockerfile.compute_worker
+      - compute_worker/**
+jobs:
+  build:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v5
+      - name: Build Image
+        run: docker build -t codalab/competitions-v2-compute-worker:${{ github.ref_name }} -f Dockerfile.compute_worker .
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push image with prod release tag
+        run: docker push codalab/competitions-v2-compute-worker:${{ github.ref_name }}
+      - name: Change tag to latest
+        run: |
+          docker tag codalab/competitions-v2-compute-worker:${{ github.ref_name }} codalab/competitions-v2-compute-worker:latest
+          docker push codalab/competitions-v2-compute-worker:latest

Dockerfile.compute_worker

@@ -1,34 +1,34 @@
-FROM --platform=linux/amd64 fedora:42
+FROM --platform=linux/amd64 fedora:43
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONUNBUFFERED=1
 
-# Install Docker
-RUN dnf -y install dnf-plugins-core && \
-    dnf-3 config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && \
-    dnf -y update && \
-    dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \
+# Install Python
+RUN dnf -y update && \
     dnf install -y python3.9 && \
     dnf clean all && \
     rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 
 
 RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
 # Poetry location so future commands (below) work
-ENV PATH $PATH:/root/.local/bin
+ENV PATH=$PATH:/root/.local/bin
 # Want poetry to use system python of docker container
 RUN poetry config virtualenvs.create false
 RUN poetry config virtualenvs.in-project false
+
+
 COPY ./compute_worker/pyproject.toml ./
 COPY ./compute_worker/poetry.lock ./
 # To use python3.9 instead of system python
+
 RUN poetry config virtualenvs.prefer-active-python true && poetry install
 
 ADD compute_worker .
 COPY ./src/settings/logs_loguru.py /usr/bin
 
-CMD celery -A compute_worker worker \
-    -l info \
-    -Q compute-worker \
-    -n compute-worker@%n \
-    --concurrency=1
+# Uninstall Poetry since we don't need it anymore and it can introduce CVEs
+RUN curl -sSL https://install.python-poetry.org | python3.9 - --uninstall
+
+ENTRYPOINT ["/bin/bash", "-c"]
+CMD ["celery -A compute_worker worker -l info -Q compute-worker -n compute-worker@$HOSTNAME --concurrency=1"]