feat: new populate initial data from prod

.env.example

@@ -1,25 +1,57 @@
-# Mail notifications
-ADMINS="Gabriel Milan,gabriel.gazola@poli.ufrj.br"
-EMAIL_HOST_USER="notifications@gmail.com"
-EMAIL_HOST_PASSWORD="password"
-# Django configurations
-DJANGO_SECRET_KEY="some-secret"
-DJANGO_SETTINGS_MODULE="backend.settings.local"
+# =============================================================================
+# ESSENTIAL SETTINGS FOR DEVELOPMENT
+# =============================================================================
+
+# Django
+DJANGO_JWT_ALGORITHM=HS256
+DJANGO_SECRET_KEY=dev-secret-key-change-in-production
+DJANGO_SETTINGS_MODULE=backend.settings.local
+
+# Database (configured for Docker - do not change if using docker compose)
+DB_HOST=database
+DB_PORT=5432
+DB_NAME=postgres
+DB_USER=postgres
+DB_PASSWORD=postgres
+
+# Queue (configured for Docker - do not change if using docker compose)
+REDIS_HOST=queue
+REDIS_PORT=6379
+
+# Search Index (configured for Docker - do not change if using docker compose)
+ELASTICSEARCH_URL=http://index:9200
+
 # Logger
-LOGGER_LEVEL="DEBUG"
-LOGGER_IGNORE="faker,haystack"
-LOGGER_SERIALIZE=""
-# Database
-DB_HOST="localhost"
-DB_PORT="5432"
-DB_NAME="postgres"
-DB_USER="postgres"
-DB_PASSWORD="postgres"
-# Queue
-REDIS_HOST="localhost"
-REDIS_PORT="6379"
-# Index
-ELASTICSEARCH_URL=http://localhost:9200
-# Google
-GOOGLE_OAUTH_CLIENT_ID="google_key"
-GOOGLE_OAUTH_CLIENT_SECRET="google_secret"
+LOGGER_LEVEL=DEBUG
+LOGGER_IGNORE=faker,haystack
+LOGGER_SERIALIZE=
+
+# =============================================================================
+# OPTIONAL SETTINGS FOR DEVELOPMENT
+# =============================================================================
+
+# Mail notifications (only required if testing email sending)
+# ADMINS=Your Name,your.email@example.com
+# EMAIL_HOST_USER=your-notifications-email@gmail.com
+# EMAIL_HOST_PASSWORD=your-app-password
+
+# Google authentication
+# GOOGLE_OAUTH_CLIENT_ID="google_key"
+# GOOGLE_OAUTH_CLIENT_SECRET="google_secret"
+
+# Chatbot (only required if working with chatbot functionality)
+# Requires Google Cloud Platform credentials with BigQuery access
+# CHATBOT_CREDENTIALS=/app/credentials/service_account.json
+# BIGQUERY_PROJECT_ID=your-gcp-project
+# BASE_URL_BACKEND=http://localhost:8000
+# LANGSMITH_TRACING=true
+# LANGSMITH_API_KEY=your-langsmith-api-key
+# LANGSMITH_PROJECT=your-langsmith-project-name
+# For load testing only
+# MOCK_AGENT=false
+# CHATBOT_USER_EMAIL=example@email.com
+# CHATBOT_USER_PASSWORD=examplepassword
+
+# Local DB populating (only required to populate local database with production data)
+# METABASE_USER=your-metabase-user
+# METABASE_PASSWORD=your-metabase-password

.github/workflows/ci-python.yaml

@@ -11,7 +11,9 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Lint source code
-        uses: chartboost/ruff-action@v1
+        uses: astral-sh/ruff-action@v3
+        with:
+          version: "0.14.9"
   test:
     name: Test python
     runs-on: ubuntu-latest

.github/workflows/deploy-dev.yaml

@@ -8,9 +8,7 @@ on:
 jobs:
   deploy-dev:
     runs-on: ubuntu-latest
-    if: |
-      ${{ github.event_name == 'workflow_dispatch' }}
-      || ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     environment:
       name: development
       url: https://api.development.basedosdados.org
@@ -56,10 +54,10 @@ jobs:
             resources:
               limits:
                 cpu: 500m
-                memory: 4096Mi
+                memory: 2Gi
               requests:
                 cpu: 250m
-                memory: 1536Mi
+                memory: 1.5Gi
             envFrom:
               - secretRef:
                   name: api-development-secrets
@@ -87,8 +85,8 @@ jobs:
         run: |-
           helm upgrade \
             --wait \
-            --timeout 15m \
             --install \
+            --timeout 10m \
             --namespace website \
             --values values.yaml \
             api-dev charts/basedosdados-api/.

.github/workflows/deploy-prod.yaml

@@ -8,9 +8,7 @@ on:
 jobs:
   deploy-prod:
     runs-on: ubuntu-latest
-    if: |
-      ${{ github.event_name == 'workflow_dispatch' }}
-      || ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     environment:
       name: production
       url: https://backend.basedosdados.org
@@ -56,10 +54,10 @@ jobs:
             resources:
               limits:
                 cpu: 1000m
-                memory: 1536Mi
+                memory: 4Gi
               requests:
                 cpu: 500m
-                memory: 1024Mi
+                memory: 2Gi
             envFrom:
               - secretRef:
                   name: api-prod-secrets
@@ -88,6 +86,7 @@ jobs:
           helm upgrade \
             --wait \
             --install \
+            --timeout 10m \
             --namespace website \
             --values values.yaml \
             api-prod charts/basedosdados-api/.

.github/workflows/deploy-staging.yaml

@@ -8,9 +8,7 @@ on:
 jobs:
   deploy-staging:
     runs-on: ubuntu-latest
-    if: |
-      ${{ github.event_name == 'workflow_dispatch' }}
-      || ${{ github.event.workflow_run.conclusion == 'success' }}
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
     environment:
       name: staging
       url: https://staging.backend.basedosdados.org
@@ -56,10 +54,10 @@ jobs:
             resources:
               limits:
                 cpu: 500m
-                memory: 1Gi
+                memory: 2Gi
               requests:
                 cpu: 250m
-                memory: 500Mi
+                memory: 1.5Gi
             envFrom:
               - secretRef:
                   name: api-staging-secrets
@@ -88,6 +86,7 @@ jobs:
           helm upgrade \
             --wait \
             --install \
+            --timeout 10m \
             --namespace website \
             --values values.yaml \
             api-staging charts/basedosdados-api/.

.github/workflows/release-dev.yaml

@@ -18,7 +18,7 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push PR image
+      - name: Build and push development tagged image
         uses: docker/build-push-action@v2
         with:
           context: .

.gitignore

@@ -249,10 +249,14 @@ fabric.properties
 .DS_Store
 /backend/media/
 /backend/notebooks/
+/backend/static/
 /backend/staticfiles/
 
 # Fixture
 fixtures*
 
 # Version manager
 .tool-versions
+
+# fetch_metabase script
+metabase_data

.pre-commit-config.yaml

@@ -19,13 +19,13 @@ repos:
     hooks:
       - id: poetry-check
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.2.0
+    rev: v0.14.9
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix]
       - id: ruff-format
   - repo: https://github.com/tconbeer/sqlfmt
-    rev: v0.20.0
+    rev: v0.28.2
     hooks:
       - id: sqlfmt
         language_version: python
@@ -36,4 +36,5 @@ repos:
         name: yamlfix
         types: [yaml]
         language: system
-        entry: yamlfix --exclude "charts/**/*" .
+        entry: yamlfix
+        exclude: ^charts/

Dockerfile

@@ -1,15 +1,21 @@
 ARG PYTHON_VERSION=3.11-slim
 
-FROM python:${PYTHON_VERSION}
+FROM python:$PYTHON_VERSION
 
-# Install virtualenv and create a virtual environment
-RUN pip install --no-cache-dir -U virtualenv>=20.13.1 && virtualenv /env --python=python3.11
-ENV PATH /env/bin:$PATH
+# Set shell with pipefail for safer pipe operations
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
-# Install pip requirements
-WORKDIR /app
-COPY . .
-RUN /env/bin/pip install --no-cache-dir . && rm nginx.conf
+# Define where Poetry virtual environments will be stored
+ARG POETRY_VIRTUALENVS_PATH=/opt/pypoetry/virtualenvs
+
+# Ensure that the python output is sent straight to terminal (e.g. your container log)
+# without being first buffered and that you can see the output of your application (e.g. django logs)
+# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
+ENV PYTHONUNBUFFERED=1
+
+# Prevent Python from writing .pyc files to disc
+# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # Install make, nginx and copy configuration
 RUN apt-get update \
@@ -20,20 +26,26 @@ RUN apt-get update \
 RUN apt-get update && apt-get install -y postgresql postgresql-contrib
 COPY nginx.conf /etc/nginx/nginx.conf
 
-# Prevents Python from writing .pyc files to disc
-# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE
-ENV PYTHONDONTWRITEBYTECODE 1
+# Install Poetry and add it to PATH so its commands can be executed
+# from anywhere, without specifying the full path to its executable.
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.1.3
+ENV PATH="/root/.local/bin:$PATH"
 
-# Ensures that the python output is sent straight to terminal (e.g. your container log)
-# without being first buffered and that you can see the output of your application (e.g. django logs)
-# in real time. Equivalent to python -u: https://docs.python.org/3/using/cmdline.html#cmdoption-u
-ENV PYTHONUNBUFFERED 1
+# Create the folder where Poetry virtual environments will be stored and make it
+# accessible to all users. This is needed by the 'www-data' user during server startup
+RUN mkdir -p $POETRY_VIRTUALENVS_PATH && chmod 755 $POETRY_VIRTUALENVS_PATH
+ENV POETRY_VIRTUALENVS_PATH=$POETRY_VIRTUALENVS_PATH
+
+# Copy and install project
+WORKDIR /app
+COPY . .
+RUN poetry install --only main && rm nginx.conf
 
-# Copy app, generate static and set permissions
-RUN /env/bin/python manage.py collectstatic --no-input --settings=backend.settings.base && \
+# Generate static and set permissions
+RUN poetry run python manage.py collectstatic --no-input --settings=backend.settings.base && \
     chown -R www-data:www-data /app
 
 # Expose and run app
 EXPOSE 80
 STOPSIGNAL SIGKILL
-CMD ["/app/start-server.sh"]
+CMD ["poetry", "run", "/app/start-server.sh"]