From 617aa159c9a7d703ffe41875acf3c30765b944db Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 02:01:42 -0700 Subject: [PATCH 01/23] added devcontainer configs --- .devcontainer/devcontainer.json | 23 +++++++++++++++++++++++ .gitignore | 3 ++- 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..57dc32f --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,23 @@ +{ + "dockerComposeFile": "docker-compose.yml", + "service": "jupyterbase", + "workspaceFolder": "/apps", + "forwardPorts": [8888, 8080, 9001], + "portsAttributes": { + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + } + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 102c798..7f8f492 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,8 @@ docker-compose.override.yml **/pgduckdb_data/ **/minio_data/ -# Ignore all hidden files (except .gitignore and .github) +# Ignore all hidden files (except .devcontainer, .gitignore and .github) .* !.gitignore !.github/ +!.devcontainer/ From 4ac2ba28b8706efa44f0cd9ad9fbc6581db55921 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 02:11:06 -0700 Subject: [PATCH 02/23] added vscode configs for codespaces --- .devcontainer/devcontainer.json | 58 ++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 57dc32f..8a736e0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,23 +1,37 @@ { - "dockerComposeFile": "docker-compose.yml", - "service": "jupyterbase", - "workspaceFolder": "/apps", - "forwardPorts": [8888, 8080, 9001], - "portsAttributes": { - "8888": { - "label": "Jupyter Lab", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8080": { - "label": "pgAdmin", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "9001": { - "label": "MinIO Console", - "onAutoForward": "openBrowser", - "visibility": "public" - } - } -} \ No newline at end of file + "name":"Sonnet Scripts Dev Container", + "dockerComposeFile":[ + "../docker-compose.yml" + ], + "service":"jupyterbase", + "workspaceFolder":"/apps", + "forwardPorts":[ + 8888, + 8080, + 9001 + ], + "portsAttributes":{ + "8888":{ + "label":"Jupyter Lab", + "onAutoForward":"openBrowser", + "visibility":"public" + }, + "8080":{ + "label":"pgAdmin", + "onAutoForward":"openBrowser", + "visibility":"public" + }, + "9001":{ + "label":"MinIO Console", + "onAutoForward":"openBrowser", + "visibility":"public" + } + }, + "remoteUser":"root", + "extensions":[ + "ms-python.python", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance" + ], + "shutdownAction":"stopCompose" + } \ No newline at end of file From 05d59a24a81a61f52c8face1fcfd180f0cbd764d Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 02:24:42 -0700 Subject: [PATCH 03/23] update how docker is read for custom dockerfile --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a5236ca..4a0a2b6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: build: context: ./pythonbase container_name: pythonbase - image: pythonbase + image: pythonbase:latest depends_on: linuxbase: condition: service_completed_successfully @@ -15,7 +15,7 @@ services: build: context: ./jupyterbase container_name: jupyterbase - image: jupyterbase + image: jupyterbase:latest depends_on: pythonbase: condition: service_started @@ -28,7 +28,7 @@ services: build: context: ./pipelinebase container_name: pipelinebase - image: pipelinebase + image: pipelinebase:latest depends_on: pythonbase: condition: service_started @@ -41,7 +41,7 @@ services: build: context: ./linuxbase container_name: linuxbase - image: linuxbase + image: linuxbase:latest pgduckdb: image: pgduckdb/pgduckdb:17-v0.1.0 From c7a595e1e107a3f49438f7732b3cc9a3296b0714 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 02:49:51 -0700 Subject: [PATCH 04/23] updated devcontainer --- .devcontainer/devcontainer.json | 94 +++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8a736e0..03a30a8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,37 +1,65 @@ { - "name":"Sonnet Scripts Dev Container", - "dockerComposeFile":[ - "../docker-compose.yml" + "name": "Sonnet Scripts Dev Container", + "dockerComposeFile": ["../docker-compose.yml"], + "service": "jupyterbase", + "workspaceFolder": "/apps", + "forwardPorts": [8888, 8080, 9001], + "portsAttributes": { + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + } + }, + "remoteUser": "root", + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance" ], - "service":"jupyterbase", - "workspaceFolder":"/apps", - "forwardPorts":[ - 8888, - 8080, - 9001 + "shutdownAction": "stopCompose", + "postCreateCommand": "make rebuild-clean", + "runArgs": [ + "--cap-add=SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--privileged", + "--init", + "--label", + "CodespacesImageType=Default", + "--label", + "ContainerVersion=13", + "--cap-add", + "sys_nice", + "--network", + "host" ], - "portsAttributes":{ - "8888":{ - "label":"Jupyter Lab", - "onAutoForward":"openBrowser", - "visibility":"public" - }, - "8080":{ - "label":"pgAdmin", - "onAutoForward":"openBrowser", - "visibility":"public" - }, - "9001":{ - "label":"MinIO Console", - "onAutoForward":"openBrowser", - "visibility":"public" - } + "settings": { + "go.toolsManagement.checkForUpdates": "local", + "go.useLanguageServer": true, + "go.gopath": "/go", + "python.defaultInterpreterPath": "/venv/bin/python", + "jupyter.kernels.filter": [ + { + "path": "/venv/bin/python", + "type": "pythonEnvironment" + } + ], + "lldb.executable": "/usr/bin/lldb" }, - "remoteUser":"root", - "extensions":[ - "ms-python.python", - "ms-azuretools.vscode-docker", - "ms-python.vscode-pylance" - ], - "shutdownAction":"stopCompose" - } \ No newline at end of file + "containerEnv": { + "CODESPACES": "true", + "ContainerVersion": "13", + "RepositoryName": "sonnet-scripts" + } + } \ No newline at end of file From 3177b85a012d7065f75be23517f9268e2b9b10fb Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 03:00:34 -0700 Subject: [PATCH 05/23] attempting docker compose labels --- .devcontainer/devcontainer.json | 65 --------------------------------- docker-compose.yml | 12 ++++++ 2 files changed, 12 insertions(+), 65 deletions(-) delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 03a30a8..0000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "name": "Sonnet Scripts Dev Container", - "dockerComposeFile": ["../docker-compose.yml"], - "service": "jupyterbase", - "workspaceFolder": "/apps", - "forwardPorts": [8888, 8080, 9001], - "portsAttributes": { - "8888": { - "label": "Jupyter Lab", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8080": { - "label": "pgAdmin", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "9001": { - "label": "MinIO Console", - "onAutoForward": "openBrowser", - "visibility": "public" - } - }, - "remoteUser": "root", - "extensions": [ - "ms-python.python", - "ms-azuretools.vscode-docker", - "ms-python.vscode-pylance" - ], - "shutdownAction": "stopCompose", - "postCreateCommand": "make rebuild-clean", - "runArgs": [ - "--cap-add=SYS_PTRACE", - "--security-opt", - "seccomp=unconfined", - "--privileged", - "--init", - "--label", - "CodespacesImageType=Default", - "--label", - "ContainerVersion=13", - "--cap-add", - "sys_nice", - "--network", - "host" - ], - "settings": { - "go.toolsManagement.checkForUpdates": "local", - "go.useLanguageServer": true, - "go.gopath": "/go", - "python.defaultInterpreterPath": "/venv/bin/python", - "jupyter.kernels.filter": [ - { - "path": "/venv/bin/python", - "type": "pythonEnvironment" - } - ], - "lldb.executable": "/usr/bin/lldb" - }, - "containerEnv": { - "CODESPACES": "true", - "ContainerVersion": "13", - "RepositoryName": "sonnet-scripts" - } - } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4a0a2b6..0f0aa91 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,10 @@ services: condition: service_healthy ports: - "8888:8888" + labels: + - "devcontainer.ports.8888.label=Jupyter Lab" + - "devcontainer.ports.8888.onAutoForward=openBrowser" + - "devcontainer.ports.8888.visibility=public" pipelinebase: build: @@ -70,6 +74,10 @@ services: PGADMIN_CONFIG_CONSOLE_LOG_LEVEL: 10 ports: - "8080:80" + labels: + - "devcontainer.ports.8080.label=pgAdmin" + - "devcontainer.ports.8080.onAutoForward=openBrowser" + - "devcontainer.ports.8080.visibility=public" volumes: - ./pgadmin/servers.json:/pgadmin4/servers.json - ./pgadmin/preferences.json:/pgadmin4/preferences.json @@ -100,6 +108,10 @@ services: ports: - 9001:9001 - 9000:9000 + labels: + - "devcontainer.ports.9001.label=MinIO Console" + - "devcontainer.ports.9001.onAutoForward=openBrowser" + - "devcontainer.ports.9001.visibility=public" command: ["server", "/data", "--console-address", ":9001"] cloudbeaver: From ba8dfc372611a9ea0fcb853bf9cbc579110e64b4 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 03:13:58 -0700 Subject: [PATCH 06/23] trying devcontainer again --- .devcontainer/devcontainer.json | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..ae480d8 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,18 @@ +{ + "name": "Sonnet Scripts - Codespaces Dev Container", + "dockerComposeFile": [ + "../docker-compose.yml" + ], + "service": "apps/", // Replace "app" with the name of your main service in the compose file + "workspaceFolder": "/workspaces/sonnet-scripts", + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ], + "forwardPorts": [], + "postCreateCommand": "", + "remoteUser": "vscode" +} \ No newline at end of file From 1ea7602e005dc5f4ca8c84df89a46fe91036ed9f Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 03:23:10 -0700 Subject: [PATCH 07/23] updated devcontainer --- .devcontainer/devcontainer.json | 42 +++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ae480d8..700d47e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,16 +3,44 @@ "dockerComposeFile": [ "../docker-compose.yml" ], - "service": "apps/", // Replace "app" with the name of your main service in the compose file - "workspaceFolder": "/workspaces/sonnet-scripts", + "service": "jupyterbase", + "workspaceFolder": "/apps", "settings": { - "terminal.integrated.shell.linux": "/bin/bash" + "terminal.integrated.shell.linux": "/bin/bash", + "python.defaultInterpreterPath": "/venv/bin/python" }, "extensions": [ "ms-python.python", - "ms-azuretools.vscode-docker" + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance" ], - "forwardPorts": [], - "postCreateCommand": "", - "remoteUser": "vscode" + "forwardPorts": [8888, 8080, 9001, 8978, 5432], + "portsAttributes": { + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8978": { + "label": "CloudBeaver", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "5432": { + "label": "Postgres/DuckDB", + "visibility": "private" + } + }, + "remoteUser": "root", + "shutdownAction": "stopCompose" } \ No newline at end of file From 2751791402664b537ea3313a31bf15c48dd59952 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 03:27:49 -0700 Subject: [PATCH 08/23] again... --- .devcontainer/devcontainer.json | 65 +++++++++++++++++---------------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 700d47e..c69ca5b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,46 +1,47 @@ { "name": "Sonnet Scripts - Codespaces Dev Container", "dockerComposeFile": [ - "../docker-compose.yml" + "../docker-compose.yml" ], "service": "jupyterbase", "workspaceFolder": "/apps", "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "python.defaultInterpreterPath": "/venv/bin/python" + "terminal.integrated.shell.linux": "/bin/bash", + "python.defaultInterpreterPath": "/venv/bin/python" }, "extensions": [ - "ms-python.python", - "ms-azuretools.vscode-docker", - "ms-python.vscode-pylance" + "ms-python.python", + "ms-azuretools.vscode-docker", + "ms-python.vscode-pylance" ], "forwardPorts": [8888, 8080, 9001, 8978, 5432], "portsAttributes": { - "8888": { - "label": "Jupyter Lab", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8080": { - "label": "pgAdmin", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "9001": { - "label": "MinIO Console", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8978": { - "label": "CloudBeaver", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "5432": { - "label": "Postgres/DuckDB", - "visibility": "private" - } + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8978": { + "label": "CloudBeaver", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "5432": { + "label": "Postgres/DuckDB", + "visibility": "private" + } }, "remoteUser": "root", - "shutdownAction": "stopCompose" -} \ No newline at end of file + "shutdownAction": "stopCompose", + "postCreateCommand": "make rebuild-clean" + } \ No newline at end of file From 4dcc642b79329eeb35befab46443838883f7407c Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 03:34:31 -0700 Subject: [PATCH 09/23] updated path for dockercompose --- .devcontainer/devcontainer.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c69ca5b..ef7ec87 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,6 @@ { "name": "Sonnet Scripts - Codespaces Dev Container", - "dockerComposeFile": [ - "../docker-compose.yml" - ], + "dockerComposeFile": ["docker-compose.yml"], "service": "jupyterbase", "workspaceFolder": "/apps", "settings": { From 242c252fc20bfd6c441a9ffe41eab2f9f14ea979 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:24:42 -0700 Subject: [PATCH 10/23] updates to improve codespaces build --- .devcontainer/devcontainer.json | 64 +++++++++++---------------------- docker-compose.yml | 51 +++++++++++--------------- pipelinebase/Dockerfile | 12 ++++--- 3 files changed, 47 insertions(+), 80 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ef7ec87..981baf5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,45 +1,21 @@ { - "name": "Sonnet Scripts - Codespaces Dev Container", - "dockerComposeFile": ["docker-compose.yml"], - "service": "jupyterbase", - "workspaceFolder": "/apps", - "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "python.defaultInterpreterPath": "/venv/bin/python" - }, - "extensions": [ - "ms-python.python", - "ms-azuretools.vscode-docker", - "ms-python.vscode-pylance" - ], - "forwardPorts": [8888, 8080, 9001, 8978, 5432], - "portsAttributes": { - "8888": { - "label": "Jupyter Lab", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8080": { - "label": "pgAdmin", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "9001": { - "label": "MinIO Console", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "8978": { - "label": "CloudBeaver", - "onAutoForward": "openBrowser", - "visibility": "public" - }, - "5432": { - "label": "Postgres/DuckDB", - "visibility": "private" - } - }, - "remoteUser": "root", - "shutdownAction": "stopCompose", - "postCreateCommand": "make rebuild-clean" - } \ No newline at end of file + "name": "Sonnet Scripts Dev", + "dockerComposeFile": [ + "../docker-compose.yml" + ], + "service": "jupyterbase", + "workspaceFolder": "/apps", + "build": { + "target": "jupyterbase" + }, + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker" + ], + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python" + }, + "remoteEnv": {}, + "shutdownAction": "stopCompose", + "initializeCommand": "docker compose --profile build build" +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 0f0aa91..773a201 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,11 @@ services: + linuxbase: + build: + context: ./linuxbase + container_name: linuxbase + image: linuxbase:latest + pythonbase: build: context: ./pythonbase @@ -23,10 +29,7 @@ services: condition: service_healthy ports: - "8888:8888" - labels: - - "devcontainer.ports.8888.label=Jupyter Lab" - - "devcontainer.ports.8888.onAutoForward=openBrowser" - - "devcontainer.ports.8888.visibility=public" + profiles: ["build"] pipelinebase: build: @@ -41,12 +44,6 @@ services: minio: condition: service_started - linuxbase: - build: - context: ./linuxbase - container_name: linuxbase - image: linuxbase:latest - pgduckdb: image: pgduckdb/pgduckdb:17-v0.1.0 container_name: pgduckdb @@ -74,10 +71,6 @@ services: PGADMIN_CONFIG_CONSOLE_LOG_LEVEL: 10 ports: - "8080:80" - labels: - - "devcontainer.ports.8080.label=pgAdmin" - - "devcontainer.ports.8080.onAutoForward=openBrowser" - - "devcontainer.ports.8080.visibility=public" volumes: - ./pgadmin/servers.json:/pgadmin4/servers.json - ./pgadmin/preferences.json:/pgadmin4/preferences.json @@ -108,28 +101,24 @@ services: ports: - 9001:9001 - 9000:9000 - labels: - - "devcontainer.ports.9001.label=MinIO Console" - - "devcontainer.ports.9001.onAutoForward=openBrowser" - - "devcontainer.ports.9001.visibility=public" command: ["server", "/data", "--console-address", ":9001"] - cloudbeaver: - image: dbeaver/cloudbeaver:25.0.3 - container_name: cloudbeaver - ports: - - "8978:8978" - volumes: - - cloudbeaver-data:/opt/cloudbeaver/workspace - - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf - - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf - depends_on: - - pgduckdb + # cloudbeaver: + # image: dbeaver/cloudbeaver:25.0.3 + # container_name: cloudbeaver + # ports: + # - "8978:8978" + # volumes: + # - cloudbeaver-data:/opt/cloudbeaver/workspace + # - ./cloudbeaver/conf/.cloudbeaver.auto.conf:/opt/cloudbeaver/conf/.cloudbeaver.auto.conf + # - ./cloudbeaver/conf/.cloudbeaver.runtime.conf:/opt/cloudbeaver/workspace/.data/.cloudbeaver.runtime.conf + # depends_on: + # - pgduckdb volumes: pgduckdb_data: driver: local data: driver: local - cloudbeaver-data: - driver: local \ No newline at end of file + # cloudbeaver-data: + # driver: local \ No newline at end of file diff --git a/pipelinebase/Dockerfile b/pipelinebase/Dockerfile index fb1b5ea..e3c22a2 100644 --- a/pipelinebase/Dockerfile +++ b/pipelinebase/Dockerfile @@ -8,15 +8,17 @@ RUN uv venv --python 3.12.6 /venv && \ echo "\nsource /venv/bin/activate\n" >> /root/.zshrc && \ uv --version -# Copy application files -COPY etl_pipelines /apps/etl_pipelines -COPY ingest_claims /apps/ingest_claims -COPY tests /apps/tests +# Copy requirements first to leverage Docker cache COPY requirements.txt /apps/requirements.txt # Install requirements RUN . /venv/bin/activate && \ - uv pip install --upgrade -r /apps/requirements.txt + uv pip install -r /apps/requirements.txt + +# Copy application files +COPY etl_pipelines /apps/etl_pipelines +COPY ingest_claims /apps/ingest_claims +COPY tests /apps/tests # Create empty __init__.py files to ensure proper module structure RUN touch /apps/__init__.py From 2e24dba9d3b0d5eb550f8f835ff2ed2193417cb8 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:32:40 -0700 Subject: [PATCH 11/23] remove upgrades flags --- jupyterbase/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyterbase/Dockerfile b/jupyterbase/Dockerfile index 7270580..a1252fe 100644 --- a/jupyterbase/Dockerfile +++ b/jupyterbase/Dockerfile @@ -11,7 +11,7 @@ RUN uv venv --python 3.12.6 /venv && \ # Copy and install Python dependencies COPY requirements.txt /apps/requirements.txt RUN . /venv/bin/activate && \ - uv pip install --upgrade -r /apps/requirements.txt + uv pip install -r /apps/requirements.txt # Ensure module structure (optional but recommended) RUN touch /apps/__init__.py From d20b1d4b986dd7b0afa9c8517bc9bcf7588ad7b2 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:39:53 -0700 Subject: [PATCH 12/23] removed auto build flag --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 981baf5..81850e4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,5 +17,4 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", - "initializeCommand": "docker compose --profile build build" } \ No newline at end of file From 016e305448388ee466847c88efaa0d704be4aac6 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:41:37 -0700 Subject: [PATCH 13/23] "initializeCommand": "docker compose --profile build build is needed" --- .devcontainer/devcontainer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 81850e4..981baf5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,4 +17,5 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", + "initializeCommand": "docker compose --profile build build" } \ No newline at end of file From 8d1e453a8abc28dbb18f5052ac1d663b4d0a3f30 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:45:40 -0700 Subject: [PATCH 14/23] update build call --- .devcontainer/devcontainer.json | 2 +- docker-compose.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 981baf5..19ad308 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,5 +17,5 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", - "initializeCommand": "docker compose --profile build build" + "initializeCommand": "docker compose build pythonbase" } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 773a201..1233d9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,7 +29,6 @@ services: condition: service_healthy ports: - "8888:8888" - profiles: ["build"] pipelinebase: build: From c88fda554e5c5b5ec33d54a7754e4374760635bc Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:47:20 -0700 Subject: [PATCH 15/23] update to build linuxbase first --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 19ad308..0402843 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,5 +17,5 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", - "initializeCommand": "docker compose build pythonbase" + "initializeCommand": "docker compose build linuxbase" } \ No newline at end of file From a7233867766a1d72f65f8807733ed00d93766c7c Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:51:12 -0700 Subject: [PATCH 16/23] updated build commands --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0402843..e755d08 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,5 +17,5 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", - "initializeCommand": "docker compose build linuxbase" + "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase" } \ No newline at end of file From 50eb6dbe39313e0c62e0bce769cab929a24a558e Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 04:57:57 -0700 Subject: [PATCH 17/23] updated workspace folder --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e755d08..bf59bee 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "../docker-compose.yml" ], "service": "jupyterbase", - "workspaceFolder": "/apps", + "workspaceFolder": "/workspace", "build": { "target": "jupyterbase" }, From 73a5f05670f52da4a4c92c88609930fab0d67a3e Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 05:04:34 -0700 Subject: [PATCH 18/23] updated configs --- .devcontainer/devcontainer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bf59bee..f5e18b9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,10 +3,10 @@ "dockerComposeFile": [ "../docker-compose.yml" ], - "service": "jupyterbase", - "workspaceFolder": "/workspace", + "service": "pythonbase", + "workspaceFolder": "/workspaces/sonnet-scripts", "build": { - "target": "jupyterbase" + "target": "pythonbase" }, "extensions": [ "ms-python.python", From 613fc01c1bc4788f3438480d640eef71c75941ec Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 05:13:55 -0700 Subject: [PATCH 19/23] updated pythonbase dockerfile --- pythonbase/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonbase/Dockerfile b/pythonbase/Dockerfile index db3d715..83d4952 100644 --- a/pythonbase/Dockerfile +++ b/pythonbase/Dockerfile @@ -11,7 +11,7 @@ ENV PATH=/venv/bin:$PATH \ PATH="/root/.local/bin/:$PATH" # Install uv, curl, and the PostgreSQL client (`psql`) -RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip +RUN apt-get update && apt-get install -y ca-certificates curl postgresql-client unzip make # Download UV and MinIO Client (mc) based on architecture RUN if [ "$TARGETARCH" = "amd64" ]; then \ @@ -64,4 +64,4 @@ RUN . /venv/bin/activate && \ RUN echo "We're good:" && \ /venv/bin/python --version -CMD ["tail", "-f", "/dev/null"] \ No newline at end of file +CMD ["/bin/bash"] \ No newline at end of file From f84c520c12b437028ed7e0aca78f32d0a5ab1305 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 05:22:38 -0700 Subject: [PATCH 20/23] more config chnages --- .devcontainer/devcontainer.json | 3 ++- pythonbase/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f5e18b9..dc5eb0a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,5 +17,6 @@ }, "remoteEnv": {}, "shutdownAction": "stopCompose", - "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase" + "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase", + "postStartCommand": "pkill -f 'tail -f /dev/null' || true" } \ No newline at end of file diff --git a/pythonbase/Dockerfile b/pythonbase/Dockerfile index 83d4952..ea48a58 100644 --- a/pythonbase/Dockerfile +++ b/pythonbase/Dockerfile @@ -64,4 +64,4 @@ RUN . /venv/bin/activate && \ RUN echo "We're good:" && \ /venv/bin/python --version -CMD ["/bin/bash"] \ No newline at end of file +CMD ["tail", "-f", "/dev/null"] \ No newline at end of file From dc1fa4462d6fa65bf70316c9b3a6653747b797ba Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 05:29:55 -0700 Subject: [PATCH 21/23] added new pythonbase cmd --- pythonbase/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonbase/Dockerfile b/pythonbase/Dockerfile index ea48a58..7b684e8 100644 --- a/pythonbase/Dockerfile +++ b/pythonbase/Dockerfile @@ -64,4 +64,4 @@ RUN . /venv/bin/activate && \ RUN echo "We're good:" && \ /venv/bin/python --version -CMD ["tail", "-f", "/dev/null"] \ No newline at end of file +CMD ["/bin/bash", "-c", "trap 'exit 0' SIGTERM; while true; do sleep 1; done"] \ No newline at end of file From f36f5579376bdbc70f03a78903ef36125b6bc513 Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 05:39:16 -0700 Subject: [PATCH 22/23] added docker to codespaces devcontainer --- .devcontainer/devcontainer.json | 7 ++++++- pythonbase/Dockerfile | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index dc5eb0a..0966165 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,5 +18,10 @@ "remoteEnv": {}, "shutdownAction": "stopCompose", "initializeCommand": "docker compose build linuxbase && docker compose build pythonbase", - "postStartCommand": "pkill -f 'tail -f /dev/null' || true" + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "version": "latest", + "moby": true + } + } } \ No newline at end of file diff --git a/pythonbase/Dockerfile b/pythonbase/Dockerfile index 7b684e8..ea48a58 100644 --- a/pythonbase/Dockerfile +++ b/pythonbase/Dockerfile @@ -64,4 +64,4 @@ RUN . /venv/bin/activate && \ RUN echo "We're good:" && \ /venv/bin/python --version -CMD ["/bin/bash", "-c", "trap 'exit 0' SIGTERM; while true; do sleep 1; done"] \ No newline at end of file +CMD ["tail", "-f", "/dev/null"] \ No newline at end of file From f9dd43d4a37357d8c8c3c2b92741a719c5dcf4ea Mon Sep 17 00:00:00 2001 From: On the Mark Data Date: Sun, 18 May 2025 06:06:10 -0700 Subject: [PATCH 23/23] port forwarding in devcontainer --- .devcontainer/devcontainer.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0966165..cd40b67 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,5 +23,32 @@ "version": "latest", "moby": true } + }, + "forwardPorts": [8888, 8080, 9001, 8978, 5432], + "portsAttributes": { + "8888": { + "label": "Jupyter Lab", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8080": { + "label": "pgAdmin", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "9001": { + "label": "MinIO Console", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "8978": { + "label": "CloudBeaver", + "onAutoForward": "openBrowser", + "visibility": "public" + }, + "5432": { + "label": "Postgres/DuckDB", + "visibility": "private" + } } } \ No newline at end of file