From 40a871a49a7a9a034e4a3b0f3a636db616909e8f Mon Sep 17 00:00:00 2001 From: OlegWock Date: Wed, 3 Dec 2025 10:37:37 +0100 Subject: [PATCH] chore: Update setup for local development for Deepnote engineers --- CONTRIBUTING.md | 67 ++++--------------- .../jupyter-for-local-hotreload/Dockerfile | 58 ++++++++++------ .../jupyter-for-local-hotreload/entrypoint.sh | 28 ++++++++ .../run-installer.sh | 42 ------------ 4 files changed, 81 insertions(+), 114 deletions(-) create mode 100644 dockerfiles/jupyter-for-local-hotreload/entrypoint.sh delete mode 100755 dockerfiles/jupyter-for-local-hotreload/run-installer.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df55851..484a4a0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,64 +151,23 @@ $ ./bin/test-local ## Development Workflow for Deepnote maintainers -### Using in Deepnote Projects +### Local Toolkit Development with Webapp -When you push a commit, a new version of `deepnote/jupyter-for-local` is built with your commit hash (shortened!). Use it in projects by updating `common.yml`: +To develop toolkit changes against a local webapp with hot-reload: -```yaml -jupyter: - image: "deepnote/jupyter-for-local:SHORTENED_COMMIT_SHA" -``` - -Alternatively, to develop against a local copy of Deepnote Toolkit, first run this command to build the image: - -```bash -docker build \ - --build-arg "FROM_PYTHON_TAG=3.11" \ - -t deepnote/deepnote-toolkit-local-hotreload \ - -f ./dockerfiles/jupyter-for-local-hotreload/Dockerfile . -``` - -Then start the container: - -```bash -# To include server logs in the output add this argument -# -e WITH_SERVER_LOGS=1 \ - -# Some toolkit features (e.g. feature flags support) require -# DEEPNOTE_PROJECT_ID to be set to work correctly. Add this -# argument with your project id -# -e DEEPNOTE_PROJECT_ID=981af2c1-fe8b-41b7-94bf-006b74cf0641 \ - -docker run \ - -v "$(pwd)":/deepnote-toolkit \ - -v /tmp/deepnote-mounts:/deepnote-mounts:shared \ - -p 8888:8888 \ - -p 2087:2087 \ - -p 8051:8051 \ - -w /deepnote-toolkit \ - --add-host=localstack.dev.deepnote.org:host-gateway \ - --rm \ - --name deepnote-toolkit-local-hotreload-container \ - deepnote/deepnote-toolkit-local-hotreload -``` - -This will start a container with Deepnote Toolkit mounted inside and expose all required ports. If you change code that runs in the kernel (e.g. you updated the DataFrame formatter), you only need to restart the kernel from Deepnote's UI. If you update code that starts Jupyter itself, you need to restart the container. And if you add or modify dependencies you need to rebuild the image. - -Now, you need to modify `common.yml` in the Deepnote app. First, replace `jupyter` service with noop image: +1. Build the local development image: + ```bash + docker build -t deepnote/jupyter-for-local:local -f ./dockerfiles/jupyter-for-local-hotreload/Dockerfile . + ``` -```yml -jupyter: - image: 'screwdrivercd/noop-container' -``` +2. Setup `DEEPNOTE_TOOLKIT_SOURCE_PATH` env variable pointing to folder with toolkit source. This can go either in `.zshrc` (or similar file for your shell) or set per shell session with `export DEEPNOTE_TOOLKIT_SOURCE_PATH=...`. If not set, webapp will try to resolve it to `../deepnote-toolkit` relative to webapp root folder. -And change `JUPYTER_HOST` variable of executor to point to host machine: +3. In the webapp repository, run: + ```bash + pnpm dev:app:local-toolkit + ``` -```yml -executor: - environment: - JUPYTER_HOST: host.docker.internal -``` +This mounts your toolkit source into the container and installs it in editable mode. Toolkit module code changes are reflected after kernel restart (use "Restart kernel" action in the webapp). ### Review Applications @@ -229,6 +188,8 @@ We use Docker to ensure reproducible environments due to Jupyter libraries' bina - `jupyter-for-local.Dockerfile`: Creates development environment with Jupyter integration, used for local development from docker-compose used in main monorepo. +- `jupyter-for-local-hotreload.Dockerfile`: Creates development environment which expectes toolkit source to be mounted at `/toolkit`. Used for development in main monorepo. + ### Production Releases To release a new version to production: diff --git a/dockerfiles/jupyter-for-local-hotreload/Dockerfile b/dockerfiles/jupyter-for-local-hotreload/Dockerfile index 4d25eb9..bac4373 100644 --- a/dockerfiles/jupyter-for-local-hotreload/Dockerfile +++ b/dockerfiles/jupyter-for-local-hotreload/Dockerfile @@ -1,32 +1,52 @@ -ARG FROM_PYTHON_TAG +# Dockerfile for local development with hot-reload support +# This container expects the toolkit source to be mounted at /toolkit +# and installs it in editable mode for live code changes +# +# Build with: +# docker build -t deepnote/jupyter-for-local:local -f dockerfiles/jupyter-for-local-hotreload/Dockerfile . + +ARG FROM_PYTHON_TAG=3.12 FROM deepnote/python:${FROM_PYTHON_TAG} +ARG FROM_PYTHON_TAG + +ENV DEBIAN_FRONTEND=noninteractive + +# Install system dependencies RUN apt-get update && \ - apt-get install -y openjdk-17-jdk && \ + apt-get install --no-install-recommends -y \ + rsync \ + git \ + # Required for pymssql + freetds-dev \ + # Required for database connectivity through ODBC + unixodbc-dev \ + # Required for secure connections (SSL/TLS) + libssl-dev && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN pip install poetry==2.2.0 +# Install Poetry and required plugins +RUN pip install --no-cache-dir poetry==2.2.0 && \ + poetry self add 'poetry-dynamic-versioning[plugin]>=1.0.0,<2.0.0' -WORKDIR /deepnote-toolkit +# Configure Poetry to create virtualenv outside the mounted source directory +RUN poetry config virtualenvs.path /opt/venvs -ENV POETRY_NO_INTERACTION=1 \ - POETRY_VIRTUALENVS_CREATE=1 \ - POETRY_VIRTUALENVS_IN_PROJECT=0 \ - POETRY_CACHE_DIR=/tmp/poetry_cache +# Create toolkit directory (will be mounted over, but needed for initial setup) +RUN mkdir -p /toolkit /opt/venvs -COPY pyproject.toml poetry.lock poetry.toml ./ +WORKDIR /toolkit -RUN poetry install --no-interaction --no-ansi --with server --with dev +# Environment variables for development mode +ENV DEEPNOTE_RUNNING_IN_DEV_MODE=true \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -ENV PYTHONPATH=/deepnote-toolkit:/deepnote-toolkit/installer:$PYTHONPATH \ - TOOLKIT_BUNDLE_PATH=/deepnote-toolkit \ - TOOLKIT_VERSION="local-build" \ - USERNAME=user \ - PASSWORD=password \ - DEEPNOTE_RUNNING_IN_DEV_MODE=true \ - DEEPNOTE_WEBAPP_URL="http://host.docker.internal:3002" +# Copy the entrypoint script +COPY dockerfiles/jupyter-for-local-hotreload/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh -COPY dockerfiles/jupyter-for-local-hotreload/run-installer.sh /usr/local/bin/run-installer.sh +EXPOSE 8888 -ENTRYPOINT ["/usr/local/bin/run-installer.sh"] +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dockerfiles/jupyter-for-local-hotreload/entrypoint.sh b/dockerfiles/jupyter-for-local-hotreload/entrypoint.sh new file mode 100644 index 0000000..419a66c --- /dev/null +++ b/dockerfiles/jupyter-for-local-hotreload/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +# Entrypoint script for local development container +# Installs toolkit in editable mode and starts servers via deepnote-toolkit CLI + +echo "[local-toolkit] Starting local development environment..." + +# Check if toolkit source is mounted +if [ ! -f "/toolkit/pyproject.toml" ]; then + echo "[local-toolkit] ERROR: Toolkit source not found at /toolkit" + echo "[local-toolkit] Make sure to mount the deepnote-toolkit directory to /toolkit" + exit 1 +fi + +cd /toolkit + +# Mark git directory as safe (needed for poetry-dynamic-versioning) +git config --global --add safe.directory /toolkit + +# Install dependencies and toolkit in editable mode +echo "[local-toolkit] Installing toolkit in editable mode..." +poetry install --extras server --no-interaction + +echo "[local-toolkit] Starting servers..." + +# Start servers using the toolkit CLI (handles Jupyter, LSP, config, etc.) +exec poetry run deepnote-toolkit server "$@" diff --git a/dockerfiles/jupyter-for-local-hotreload/run-installer.sh b/dockerfiles/jupyter-for-local-hotreload/run-installer.sh deleted file mode 100755 index 450f28f..0000000 --- a/dockerfiles/jupyter-for-local-hotreload/run-installer.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -while [ ! -d /deepnote-mounts/s3fs ]; do - echo "Waiting for /deepnote-mounts/s3fs to be created..." - sleep 2 -done - -mkdir -p /datasets -ln -sf /deepnote-mounts/s3fs /datasets/_deepnote_work -mkdir -p /var/log/deepnote -touch /var/log/deepnote/helpers.log - -handle_sigint() { - echo "Received SIGINT, shutting down..." - # Send TERM to all processes in our process group except ourselves - echo "Sending TERM to all remaining child processes..." - pkill -P $$ 2>/dev/null - - # Final cleanup - forcefully kill any remaining child processes - sleep 2 - echo "Sending KILL to any remaining processes..." - pkill -9 -P $$ 2>/dev/null - exit 0 -} - -trap handle_sigint SIGINT - -echo "Starting installer..." -if [ -n "${WITH_SERVER_LOGS:-}" ]; then - poetry run python -m installer --venv-path "$(poetry env info --path)" & -else - poetry run python -m installer --venv-path "$(poetry env info --path)" > /dev/null 2>&1 & -fi -installer_pid=$! - -echo "Starting log tail..." -tail -f /var/log/deepnote/helpers.log & -tail_pid=$! - -wait $installer_pid $tail_pid