diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..cd40b67 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,54 @@ +{ + "name": "Sonnet Scripts Dev", + "dockerComposeFile": [ + "../docker-compose.yml" + ], + "service": "pythonbase", + "workspaceFolder": "/workspaces/sonnet-scripts", + "build": { + "target": "pythonbase" + }, + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python" + }, + "remoteEnv": {}, + "shutdownAction": "stopCompose", + "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "latest", + "moby": true + } + }, + "forwardPorts": [8888, 8080, 9001, 8978, 5432], + "portsAttributes": { + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8978": { + "label": "CloudBeaver", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "5432": { + "label": "Postgres/DuckDB", + "visibility": "private" + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 102c798..7f8f492 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,8 @@ docker-compose.override.yml **/pgduckdb_data/ **/minio_data/ -# Ignore all hidden files (except .gitignore and .github) +# Ignore all hidden files (except .devcontainer, .gitignore and .github) .* !.gitignore !.github/ +!.devcontainer/ diff --git a/docker-compose.yml b/docker-compose.yml index a5236ca..1233d9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,16 @@ services: + linuxbase: + build: + context: ./linuxbase + container_name: linuxbase + image: linuxbase:latest + pythonbase: build: context: ./pythonbase container_name: pythonbase - image: pythonbase + image: pythonbase:latest depends_on: linuxbase: condition: service_completed_successfully @@ -15,7 +21,7 @@ services: build: context: ./jupyterbase container_name: jupyterbase - image: jupyterbase + image: jupyterbase:latest depends_on: pythonbase: condition: service_started @@ -28,7 +34,7 @@ services: build: context: ./pipelinebase container_name: pipelinebase - image: pipelinebase + image: pipelinebase:latest depends_on: pythonbase: condition: service_started @@ -37,12 +43,6 @@ services: minio: condition: service_started - linuxbase: - build: - context: ./linuxbase - container_name: linuxbase - image: linuxbase - pgduckdb: image: pgduckdb/pgduckdb:17-v0.1.0 container_name: pgduckdb @@ -102,22 +102,22 @@ services: - 9000:9000 command: ["server", "/data", "--console-address", ":9001"] - cloudbeaver: - image: dbeaver/cloudbeaver:25.0.3 - container_name: cloudbeaver - ports: - - "8978:8978" - volumes: - - cloudbeaver-data:/opt/cloudbeaver/workspace - - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf - - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf - depends_on: - - pgduckdb + # cloudbeaver: + # image: dbeaver/cloudbeaver:25.0.3 + # container_name: cloudbeaver + # ports: + # - "8978:8978" + # volumes: + # - cloudbeaver-data:/opt/cloudbeaver/workspace + # - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf + # - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf + # depends_on: + # - pgduckdb volumes: pgduckdb_data: driver: local data: driver: local - cloudbeaver-data: - driver: local \ No newline at end of file + # cloudbeaver-data: + # driver: local \ No newline at end of file diff --git a/jupyterbase/Dockerfile b/jupyterbase/Dockerfile index 7270580..a1252fe 100644 --- a/jupyterbase/Dockerfile +++ b/jupyterbase/Dockerfile @@ -11,7 +11,7 @@ RUN uv venv --python 3.12.6 /venv && \ # Copy and install Python dependencies COPY requirements.txt /apps/requirements.txt RUN . /venv/bin/activate && \ - uv pip install --upgrade -r /apps/requirements.txt + uv pip install -r /apps/requirements.txt # Ensure module structure (optional but recommended) RUN touch /apps/__init__.py diff --git a/pipelinebase/Dockerfile b/pipelinebase/Dockerfile index fb1b5ea..e3c22a2 100644 --- a/pipelinebase/Dockerfile +++ b/pipelinebase/Dockerfile @@ -8,15 +8,17 @@ RUN uv venv --python 3.12.6 /venv && \ echo "\nsource /venv/bin/activate\n" >> /root/.zshrc && \ uv --version -# Copy application files -COPY etl_pipelines /apps/etl_pipelines -COPY ingest_claims /apps/ingest_claims -COPY tests /apps/tests +# Copy requirements first to leverage Docker cache COPY requirements.txt /apps/requirements.txt # Install requirements RUN . /venv/bin/activate && \ - uv pip install --upgrade -r /apps/requirements.txt + uv pip install -r /apps/requirements.txt + +# Copy application files +COPY etl_pipelines /apps/etl_pipelines +COPY ingest_claims /apps/ingest_claims +COPY tests /apps/tests # Create empty __init__.py files to ensure proper module structure RUN touch /apps/__init__.py diff --git a/pythonbase/Dockerfile b/pythonbase/Dockerfile index db3d715..ea48a58 100644 --- a/pythonbase/Dockerfile +++ b/pythonbase/Dockerfile @@ -11,7 +11,7 @@ ENV PATH=/venv/bin:$PATH \ PATH="/root/.local/bin/:$PATH" # Install uv, curl, and the PostgreSQL client (`psql`) -RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip +RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip make # Download UV and MinIO Client (mc) based on architecture RUN if [ "$TARGETARCH" = "amd64" ]; then \