docker-compose.yaml

---
# -*- mode: yaml -*-
services:
  dagster_init:
    container_name: dagster_init
    image: mitodl/dagster:dev
    depends_on:
      postgres:
        condition: service_healthy
    build:
      context: dg_deployments/local
      dockerfile: Dockerfile
      secrets:
      - dbt_trino_username
      - dbt_trino_password
    command: ["dagster", "instance", "migrate"]
    environment:
      DBT_SCHEMA_SUFFIX: ${DBT_SCHEMA_SUFFIX}
      DBT_TRINO_USERNAME: ${DBT_TRINO_USERNAME}
      DBT_TRINO_PASSWORD: ${DBT_TRINO_PASSWORD}
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-lakehouse:dev"
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
      DAGSTER_ENV: dev
    volumes:
    - /var/run/docker.sock:/var/run/docker.sock
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dagster_home:/opt/dagster/dagster_home

  # This service runs dagster-webserver, which loads code from all code locations.
  dagster_webserver:
    container_name: dagster_webserver
    image: mitodl/dagster:dev
    depends_on:
      postgres:
        condition: service_healthy
      dagster_init:
        condition: service_completed_successfully
    build:
      context: dg_deployments/local
      dockerfile: Dockerfile
      secrets:
      - dbt_trino_username
      - dbt_trino_password
    restart: unless-stopped
    entrypoint: ["dagster-webserver", "-h", "0.0.0.0", "-p", "3000", "-w", "/app/workspace.yaml"]
    ports:
    - "3000:3000"
    environment:
      DBT_SCHEMA_SUFFIX: ${DBT_SCHEMA_SUFFIX}
      DBT_TRINO_USERNAME: ${DBT_TRINO_USERNAME}
      DBT_TRINO_PASSWORD: ${DBT_TRINO_PASSWORD}
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-lakehouse:dev"
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
      DAGSTER_IS_DEV_CLI: true
      DAGSTER_ENV: dev
    volumes:
    - /var/run/docker.sock:/var/run/docker.sock
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dagster_home:/opt/dagster/dagster_home

  # This service runs the dagster-daemon process
  dagster_daemon:
    container_name: dagster_daemon
    image: mitodl/dagster:dev
    depends_on:
      postgres:
        condition: service_healthy
      dagster_init:
        condition: service_completed_successfully
    build:
      context: dg_deployments/local
      dockerfile: Dockerfile
      secrets:
      - dbt_trino_username
      - dbt_trino_password
    restart: on-failure
    entrypoint: ["dagster-daemon", "run", "-w", "/app/workspace.yaml"]
    environment:
      DBT_SCHEMA_SUFFIX: ${DBT_SCHEMA_SUFFIX}
      DBT_TRINO_USERNAME: ${DBT_TRINO_USERNAME}
      DBT_TRINO_PASSWORD: ${DBT_TRINO_PASSWORD}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-lakehouse:dev"
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
      DAGSTER_IS_DEV_CLI: true
      DAGSTER_ENV: dev
    volumes:
    - /var/run/docker.sock:/var/run/docker.sock
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dagster_home:/opt/dagster/dagster_home

  # Code location: canvas
  dagster_canvas:
    container_name: dagster_canvas
    image: mitodl/dagster-canvas:dev
    build:
      context: .
      dockerfile: dg_projects/canvas/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000",
      "--module-name", "canvas.definitions"]
    ports:
    - "4000:4000"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-canvas:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/canvas/canvas:/app/canvas

  # Code location: data_platform
  dagster_data_platform:
    container_name: dagster_data_platform
    image: mitodl/dagster-data_platform:dev
    build:
      context: .
      dockerfile: dg_projects/data_platform/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4001",
      "--module-name", "data_platform.definitions"]
    ports:
    - "4001:4001"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-data_platform:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/data_platform/data_platform:/app/data_platform

  # Code location: edxorg
  dagster_edxorg:
    container_name: dagster_edxorg
    image: mitodl/dagster-edxorg:dev
    build:
      context: .
      dockerfile: dg_projects/edxorg/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4002",
      "--module-name", "edxorg.definitions"]
    ports:
    - "4002:4002"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-edxorg:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/edxorg/edxorg:/app/edxorg

  # Code location: lakehouse
  dagster_lakehouse:
    container_name: dagster_lakehouse
    image: mitodl/dagster-lakehouse:dev
    build:
      context: .
      dockerfile: dg_projects/lakehouse/Dockerfile
      secrets:
      - dbt_trino_username
      - dbt_trino_password
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4003",
      "--module-name", "lakehouse.definitions"]
    ports:
    - "4003:4003"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-lakehouse:dev"
      DAGSTER_ENVIRONMENT: qa
      DBT_SCHEMA_SUFFIX: ${DBT_SCHEMA_SUFFIX}
      DBT_TRINO_USERNAME: ${DBT_TRINO_USERNAME}
      DBT_TRINO_PASSWORD: ${DBT_TRINO_PASSWORD}
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/lakehouse/lakehouse:/app/lakehouse
    - ./src/ol_dbt:/opt/dbt

  # Code location: learning_resources
  dagster_learning_resources:
    container_name: dagster_learning_resources
    image: mitodl/dagster-learning_resources:dev
    build:
      context: .
      dockerfile: dg_projects/learning_resources/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4004",
      "--module-name", "learning_resources.definitions"]
    ports:
    - "4004:4004"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-learning_resources:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      VIDEO_SHORTS_BUCKET: ${VIDEO_SHORTS_BUCKET}
      VIDEO_SHORTS_MAX_SIZE_MB: ${VIDEO_SHORTS_MAX_SIZE_MB}
      VIDEO_SHORTS_GOOGLE_SHEETS_ID: ${VIDEO_SHORTS_GOOGLE_SHEETS_ID}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/learning_resources/learning_resources:/app/learning_resources

  # Code location: legacy_openedx
  dagster_legacy_openedx:
    container_name: dagster_legacy_openedx
    image: mitodl/dagster-legacy_openedx:dev
    build:
      context: .
      dockerfile: dg_projects/legacy_openedx/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4005",
      "--module-name", "legacy_openedx.definitions"]
    ports:
    - "4005:4005"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-legacy_openedx:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/legacy_openedx/legacy_openedx:/app/legacy_openedx

  # Code location: openedx
  dagster_openedx:
    container_name: dagster_openedx
    image: mitodl/dagster-openedx:dev
    build:
      context: .
      dockerfile: dg_projects/openedx/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4006",
      "--module-name", "openedx.definitions"]
    ports:
    - "4006:4006"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-openedx:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/openedx/openedx:/app/openedx

  # Code location: b2b_organization
  dagster_b2b_organization:
    container_name: dagster_b2b_organization
    image: mitodl/dagster-b2b_organization:dev
    build:
      context: .
      dockerfile: dg_projects/b2b_organization/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4007",
      "--module-name", "b2b_organization.definitions"]
    ports:
    - "4007:4007"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-b2b_organization:dev"
      DAGSTER_ENV: dev
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/b2b_organization/b2b_organization:/app/b2b_organization

    # Code location: student_risk_probability
  dagster_student_risk_probability:
    container_name: dagster_student_risk_probability
    image: mitodl/dagster-student_risk_probability:dev
    build:
      context: .
      dockerfile: dg_projects/student_risk_probability/Dockerfile
    entrypoint: ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4008",
      "--module-name", "student_risk_probability.definitions"]
    ports:
    - "4008:4008"
    environment:
      DAGSTER_CURRENT_IMAGE: "mitodl/dagster-student_risk_probability:dev"
      DAGSTER_ENV: dev
      DBT_SCHEMA_SUFFIX: ${DBT_SCHEMA_SUFFIX}
      GITHUB_TOKEN: ${GITHUB_TOKEN}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
      DAGSTER_PG_USERNAME: postgres
      DAGSTER_PG_PASSWORD: postgres # pragma: allowlist secret
      DAGSTER_PG_HOST: postgres
      DAGSTER_PG_DB: dagster
    volumes:
    - /tmp/io_manager_storage:/tmp/io_manager_storage
    - ./dg_projects/student_risk_probability/student_risk_probability:/app/student_risk_probability

  postgres:
    image: postgres
    command: ["postgres", "-c", "max_connections=300"]
    healthcheck:
      test: ["CMD", "pg_isready"]
      interval: 3s
      timeout: 3s
      retries: 10
    ports:
    - 5432
    environment:
      PGUSER: postgres
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres # pragma: allowlist secret
      POSTGRES_DB: dagster
    volumes:
    - pgdata:/var/lib/postgresql

secrets:
  dbt_trino_username:
    environment: DBT_TRINO_USERNAME
  dbt_trino_password:
    environment: DBT_TRINO_PASSWORD

volumes:
  pgdata: