Enable Configuration for GitHub Codespaces

.devcontainer/devcontainer.json

@@ -0,0 +1,54 @@
+{
+  "name": "Sonnet Scripts Dev",
+  "dockerComposeFile": [
+    "../docker-compose.yml"
+  ],
+  "service": "pythonbase",
+  "workspaceFolder": "/workspaces/sonnet-scripts",
+  "build": {
+    "target": "pythonbase"
+  },
+  "extensions": [
+    "ms-python.python",
+    "ms-azuretools.vscode-docker"
+  ],
+  "settings": {
+    "python.defaultInterpreterPath": "/usr/local/bin/python"
+  },
+  "remoteEnv": {},
+  "shutdownAction": "stopCompose",
+  "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase",
+  "features": {
+    "ghcr.io/devcontainers/features/docker-in-docker:2": {
+      "version": "latest",
+      "moby": true
+    }
+  },
+  "forwardPorts": [8888, 8080, 9001, 8978, 5432],
+  "portsAttributes": {
+    "8888": {
+      "label": "Jupyter Lab",
+      "onAutoForward": "openBrowser",
+      "visibility": "public"
+    },
+    "8080": {
+      "label": "pgAdmin",
+      "onAutoForward": "openBrowser",
+      "visibility": "public"
+    },
+    "9001": {
+      "label": "MinIO Console",
+      "onAutoForward": "openBrowser",
+      "visibility": "public"
+    },
+    "8978": {
+      "label": "CloudBeaver",
+      "onAutoForward": "openBrowser",
+      "visibility": "public"
+    },
+    "5432": {
+      "label": "Postgres/DuckDB",
+      "visibility": "private"
+    }
+  }
+}

.gitignore

@@ -47,7 +47,8 @@ docker-compose.override.yml
 **/pgduckdb_data/
 **/minio_data/
 
-# Ignore all hidden files (except .gitignore and .github)
+# Ignore all hidden files (except .devcontainer, .gitignore and .github)
 .*
 !.gitignore
 !.github/
+!.devcontainer/

docker-compose.yml

@@ -1,10 +1,16 @@
 services:
 
+  linuxbase:
+    build:
+      context: ./linuxbase
+    container_name: linuxbase
+    image: linuxbase:latest
+
   pythonbase:
     build:
       context: ./pythonbase
     container_name: pythonbase
-    image: pythonbase
+    image: pythonbase:latest
     depends_on:
       linuxbase:
         condition: service_completed_successfully
@@ -15,7 +21,7 @@ services:
     build:
       context: ./jupyterbase
     container_name: jupyterbase
-    image: jupyterbase
+    image: jupyterbase:latest
     depends_on:
       pythonbase:
         condition: service_started
@@ -28,7 +34,7 @@ services:
     build:
       context: ./pipelinebase
     container_name: pipelinebase
-    image: pipelinebase
+    image: pipelinebase:latest
     depends_on:
       pythonbase:
         condition: service_started
@@ -37,12 +43,6 @@ services:
       minio:
         condition: service_started
 
-  linuxbase:
-    build:
-      context: ./linuxbase
-    container_name: linuxbase
-    image: linuxbase
-
   pgduckdb:
     image: pgduckdb/pgduckdb:17-v0.1.0
     container_name: pgduckdb
@@ -102,22 +102,22 @@ services:
       - 9000:9000
     command: ["server", "/data", "--console-address", ":9001"]
 
-  cloudbeaver:
-    image: dbeaver/cloudbeaver:25.0.3
-    container_name: cloudbeaver
-    ports:
-      - "8978:8978"
-    volumes:
-      - cloudbeaver-data:/opt/cloudbeaver/workspace
-      - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf
-      - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf
-    depends_on:
-      - pgduckdb
+  # cloudbeaver:
+  #   image: dbeaver/cloudbeaver:25.0.3
+  #   container_name: cloudbeaver
+  #   ports:
+  #     - "8978:8978"
+  #   volumes:
+  #     - cloudbeaver-data:/opt/cloudbeaver/workspace
+  #     - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf
+  #     - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf
+  #   depends_on:
+  #     - pgduckdb
 
 volumes:
   pgduckdb_data:
    driver: local
   data:
    driver: local
-  cloudbeaver-data:
-    driver: local
+  # cloudbeaver-data:
+  #   driver: local

jupyterbase/Dockerfile

@@ -11,7 +11,7 @@ RUN uv venv --python 3.12.6 /venv && \
 # Copy and install Python dependencies
 COPY requirements.txt /apps/requirements.txt
 RUN . /venv/bin/activate && \
-    uv pip install --upgrade -r /apps/requirements.txt
+    uv pip install -r /apps/requirements.txt
 
 # Ensure module structure (optional but recommended)
 RUN touch /apps/__init__.py

pipelinebase/Dockerfile

@@ -8,15 +8,17 @@ RUN uv venv --python 3.12.6 /venv && \
     echo "\nsource /venv/bin/activate\n" >> /root/.zshrc && \
     uv --version
 
-# Copy application files
-COPY etl_pipelines /apps/etl_pipelines
-COPY ingest_claims /apps/ingest_claims
-COPY tests /apps/tests
+# Copy requirements first to leverage Docker cache
 COPY requirements.txt /apps/requirements.txt
 
 # Install requirements
 RUN . /venv/bin/activate && \
-    uv pip install --upgrade -r /apps/requirements.txt
+    uv pip install -r /apps/requirements.txt
+
+# Copy application files
+COPY etl_pipelines /apps/etl_pipelines
+COPY ingest_claims /apps/ingest_claims
+COPY tests /apps/tests
 
 # Create empty __init__.py files to ensure proper module structure
 RUN touch /apps/__init__.py

pythonbase/Dockerfile

@@ -11,7 +11,7 @@ ENV PATH=/venv/bin:$PATH \
     PATH="/root/.local/bin/:$PATH"
 
 # Install uv, curl, and the PostgreSQL client (`psql`)
-RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip
+RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip make
 
 # Download UV and MinIO Client (mc) based on architecture
 RUN if [ "$TARGETARCH" = "amd64" ]; then \