docker-compose.yml

---
x-airflow-common:
  &airflow-common
  platform: linux/arm64
  build:
    context: .
    dockerfile: ./airflow_services/Dockerfile
  env_file:
    - ./sm2a-local-config/.env
  environment:
    &airflow-common-env
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@airflow-db/airflow
    # THIS IS A FAKE CREDENTIAL FOR DEMONSTRATION PURPOSES
    # Generate with the following code
    # python -c 'from cryptography.fernet import Fernet; print(Fernet.generate_key())'
    AIRFLOW__CORE__FERNET_KEY: "Ly8wMU8r5K7jPy58M3GpkZbXDNyJz8HiJll3pu8DbIM="
    AIRFLOW__WEBSERVER__SECRET_KEY: "Ly8wMU8r5K7jPy58M3GpkZbXDNyJz8HiJll3pu8DbIM="
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@airflow-db/airflow
    AIRFLOW__CELERY__BROKER_URL: sqs://user:pass@celery-broker:9324/
    AIRFLOW__WEBSERVER__INSTANCE_NAME: "${AIRFLOW__WEBSERVER__INSTANCE_NAME:-VEDA-SM2A-Airflow}"
    AIRFLOW__LOGGING__LOGGING_LEVEL: DEBUG
    # Gotcha: Even though we set this to "True" in airflow.cfg, an environment variable overrides it
    AIRFLOW__CORE__LOAD_EXAMPLES: false
  volumes:
    - ./dags:/opt/airflow/dags
    - ./plugins:/opt/airflow/plugins
    - ./sm2a-local-config/local_airflow.cfg:/opt/airflow/airflow.cfg
    - ./sm2a-local-config/local_webserver_config.py:/opt/airflow/webserver_config.py
    - ./infrastructure/configuration:/opt/airflow/configuration
    - ./scripts:/opt/airflow/scripts
  user: "50000:0"
  depends_on:
    &airflow-common-depends-on
    celery-broker:
      condition: service_started
    airflow-db:
      condition: service_healthy

x-airflow-worker:
  &airflow-worker
  platform: linux/arm64
  env_file:
    - ./sm2a-local-config/.env
  build:
    context: .
    dockerfile: ./airflow_worker/Dockerfile
  environment:
    <<: *airflow-common-env
  volumes:
    - ./dags:/opt/airflow/dags
    - ./plugins:/opt/airflow/plugins
    - ./sm2a-local-config/local_airflow.cfg:/opt/airflow/airflow.cfg
    - ./infrastructure/configuration:/opt/airflow/configuration
    - ./scripts:/opt/airflow/scripts 
  user: "50000:0"
  depends_on:
    <<: *airflow-common-depends-on

services:
  airflow-db:
    image: postgres:13
    platform: linux/arm64
    environment:
      POSTGRES_USER: airflow
      POSTGRES_PASSWORD: airflow
      POSTGRES_DB: airflow
    volumes:
      - airflow-db-volume:/var/lib/postgresql/data
    healthcheck:
      test: [ "CMD", "pg_isready", "-U", "airflow" ]
      interval: 5s
      retries: 5
    restart: always

  # For environment parity, use backend that implements SQS interface
  # Doesn't have ARM64 image
  # https://github.com/roribio/alpine-sqs
  celery-broker:
    image: softwaremill/elasticmq:latest
    platform: linux/amd64
    expose:
      - 9324
      - 9325
    restart: always

  airflow-webserver:
    <<: *airflow-common
    command: webserver
    ports:
      - 8080:8080
    healthcheck:
      test:
        [
          "CMD",
          "curl",
          "--fail",
          "http://localhost:8080/health"
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  airflow-scheduler:
    <<: *airflow-common
    command: scheduler
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"'
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on


  airflow-worker:
    <<: *airflow-worker
    command: uv run airflow celery worker
    # SQS does not support worker remote control commands.
    # We will ping celery broker from the worker to test healthcheck.
    healthcheck:
     test:
        [
          "CMD",
          "curl",
          "http://celery-broker:9324/"
        ]
     interval: 35s
     timeout: 30s
     retries: 5
    environment:
      <<: *airflow-common-env
      # Required to handle warm shutdown of the celery workers properly
      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
      DUMB_INIT_SETSID: "0"
    restart: always
    depends_on:
      <<: *airflow-common-depends-on


  # AIP-40: Deferrable ("Async") Operators
  # https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=177050929
  airflow-triggerer:
    <<: *airflow-common
    command: triggerer
    healthcheck:
      test:
        [
          "CMD-SHELL",
          'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"'
        ]
      interval: 35s
      timeout: 30s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on

  airflow-cli:
    <<: *airflow-common
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - /bin/bash

volumes:
  airflow-db-volume: