docker-compose.yaml

services:
  # broker for celery
  rabbitmq:
    restart: unless-stopped
    image: docker.io/rabbitmq:3.11.5-alpine
    container_name: nomad_oasis_rabbitmq
    environment:
      - RABBITMQ_ERLANG_COOKIE=SWQOKODSQALRPCLNMEQG
      - RABBITMQ_DEFAULT_USER=rabbitmq
      - RABBITMQ_DEFAULT_PASS=rabbitmq
      - RABBITMQ_DEFAULT_VHOST=/
    volumes:
      - rabbitmq:/var/lib/rabbitmq
    healthcheck:
      test: ["CMD", "rabbitmq-diagnostics", "--silent", "--quiet", "ping"]
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 10s

  # the search engine
  elastic:
    restart: unless-stopped
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.28
    container_name: nomad_oasis_elastic
    environment:
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - discovery.type=single-node
    volumes:
      - elastic:/usr/share/elasticsearch/data
    healthcheck:
      test:
        - "CMD"
        - "curl"
        - "--fail"
        - "--silent"
        - "http://elastic:9200/_cat/health"
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 60s

  # the user data db
  mongo:
    restart: unless-stopped
    image: docker.io/mongo:5.0.6
    container_name: nomad_oasis_mongo
    environment:
      - MONGO_DATA_DIR=/data/db
      - MONGO_LOG_DIR=/dev/null
    volumes:
      - /nomad/mongo:/data/db
      - /nomad/mongo_config:/data/configdb
      - /nomad/.volumes/mongo:/backup
    command: mongod --logpath=/dev/null # --quiet
    healthcheck:
      test:
        - "CMD"
        - "mongo"
        - "mongo:27017/test"
        - "--quiet"
        - "--eval"
        - "'db.runCommand({ping:1}).ok'"
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 10s

  postgresql:
    container_name: nomad_oasis_postgresql
    environment:
      POSTGRES_PASSWORD: temporal
      POSTGRES_USER: temporal
    image: docker.io/postgres:16
    volumes:
      - postgresql:/var/lib/postgresql/data

  temporal:
    container_name: nomad_oasis_temporal
    depends_on:
      - postgresql
    environment:
      - DB=postgres12
      - DB_PORT=5432
      - POSTGRES_USER=temporal
      - POSTGRES_PWD=temporal
      - POSTGRES_SEEDS=postgresql
      - TEMPORAL_ADDRESS=temporal:7233
      - TEMPORAL_CLI_ADDRESS=temporal:7233
    image: docker.io/temporalio/auto-setup:1.28
    healthcheck:
      test: >
        bash -c '
          temporal operator cluster health --address $$(hostname -i):7233 |
          grep -q "SERVING"
        '
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 15s

  # nomad worker (processing)
  worker:
    restart: unless-stopped
    image: ghcr.io/exciting/sol-oasis:main
    platform: linux/amd64
    environment:
      NOMAD_SERVICE: nomad_oasis_worker
      NOMAD_RABBITMQ_HOST: rabbitmq
      NOMAD_ELASTIC_HOST: elastic
      NOMAD_MONGO_HOST: mongo
      NOMAD_LOGSTASH_HOST: logtransfer
      NOMAD_TEMPORAL_HOST: temporal
    env_file:
      - ./.env
    depends_on:
      temporal:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      elastic:
        condition: service_healthy
      mongo:
        condition: service_healthy
    volumes:
      - ./configs/nomad.yaml:/app/nomad.yaml
      - /nomad/.volumes/fs:/app/.volumes/fs
    command: python -m nomad.cli admin run action-internal-worker
    healthcheck:
      # See https://keithtenzer.com/temporal/Deploying_Temporal_Worker_on_Kubernetes/#liveliness-and-readiness-probes
      # for rationale on using 'ls /' as a liveness/readiness probe for Temporal workers.
      test: ls /
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 15s
    deploy:
      replicas: 2
      resources:
        limits:
          cpus: "4.0" # Maximum 4 CPU cores
          memory: 8G # Maximum 4GB RAM
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 15
        window: 1800s

  ## nomad gpu worker (actions)
  # gpu_worker:
  #   restart: unless-stopped
  #   image: ghcr.io/fairmat-nfdi/nomad-distro-template/gpu-action:main
  #   platform: linux/amd64
  #   container_name: nomad_oasis_gpu_worker
  #   env_file:
  #     - ./.env
  #   deploy:
  #     replicas: 1
  #     resources:
  #       reservations:
  #         devices:
  #           - driver: nvidia
  #             count: all # Use all available GPUs
  #             capabilities: [gpu]
  #   environment:
  #     NOMAD_SERVICE: nomad_oasis_gpu_worker
  #     NOMAD_RABBITMQ_HOST: rabbitmq
  #     NOMAD_ELASTIC_HOST: elastic
  #     NOMAD_MONGO_HOST: mongo
  #     NOMAD_LOGSTASH_HOST: logtransfer
  #     NOMAD_TEMPORAL_HOST: temporal
  #   depends_on:
  #     rabbitmq:
  #       condition: service_healthy
  #     elastic:
  #       condition: service_healthy
  #     mongo:
  #       condition: service_healthy
  #     temporal:
  #       condition: service_healthy
  #   volumes:
  #     # - ./configs/nomad.yaml:/app/nomad.yaml
  #     - ./.volumes/fs:/app/.volumes/fs
  #   command: python -m nomad.cli admin run action-gpu-worker

  # # nomad cpu worker (actions)
  # cpu_worker:
  #   restart: unless-stopped
  #   image: ghcr.io/fairmat-nfdi/nomad-distro-template/cpu-action:main
  #   platform: linux/amd64
  #   container_name: nomad_oasis_cpu_worker
  #   env_file:
  #     - ./.env
  #   environment:
  #     NOMAD_SERVICE: nomad_oasis_cpu_worker
  #     NOMAD_RABBITMQ_HOST: rabbitmq
  #     NOMAD_ELASTIC_HOST: elastic
  #     NOMAD_MONGO_HOST: mongo
  #     NOMAD_LOGSTASH_HOST: logtransfer
  #     NOMAD_TEMPORAL_HOST: temporal
  #   deploy:
  #     replicas: 1
  #     resources:
  #       limits:
  #         cpus: "2.0" # Maximum 2 CPU cores
  #         memory: 8G # Maximum 8GB RAM
  #   depends_on:
  #     rabbitmq:
  #       condition: service_healthy
  #     elastic:
  #       condition: service_healthy
  #     mongo:
  #       condition: service_healthy
  #     temporal:
  #       condition: service_healthy
  #   volumes:
  #     # - ./configs/nomad.yaml:/app/nomad.yaml
  #     - ./.volumes/fs:/app/.volumes/fs
  #   command: python -m nomad.cli admin run action-cpu-worker

  # nomad app (api + proxy)
  app:
    restart: unless-stopped
    image: ghcr.io/exciting/sol-oasis:main
    platform: linux/amd64
    container_name: nomad_oasis_app
    environment:
      NOMAD_SERVICE: nomad_oasis_app
      NOMAD_SERVICES_API_PORT: 80
      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
      NOMAD_RABBITMQ_HOST: rabbitmq
      NOMAD_ELASTIC_HOST: elastic
      NOMAD_MONGO_HOST: mongo
      NOMAD_LOGSTASH_HOST: logtransfer
      NOMAD_NORTH_HUB_HOST: north
      NOMAD_TEMPORAL_HOST: temporal
    env_file:
      - ./.env
    depends_on:
      temporal:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      elastic:
        condition: service_healthy
      mongo:
        condition: service_healthy
      north:
        condition: service_started
    volumes:
      - ./configs/nomad.yaml:/app/nomad.yaml
      - /nomad/.volumes/fs:/app/.volumes/fs
    command: ./run.sh
    healthcheck:
      test:
        - "CMD"
        - "curl"
        - "--fail"
        - "--silent"
        - "http://localhost:8000/-/health"
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 10s

  # nomad remote tools hub (JupyterHUB, e.g. for AI Toolkit)
  north:
    restart: unless-stopped
    image: ghcr.io/exciting/sol-oasis:main
    platform: linux/amd64
    container_name: nomad_oasis_north
    environment:
      NOMAD_SERVICE: nomad_oasis_north
      NOMAD_NORTH_DOCKER_NETWORK: nomad_oasis_network
      NOMAD_NORTH_HUB_CONNECT_IP: north
      NOMAD_NORTH_HUB_IP: "0.0.0.0"
      NOMAD_NORTH_HUB_HOST: north
      NOMAD_SERVICES_API_HOST: app
      NOMAD_FS_EXTERNAL_WORKING_DIRECTORY: "$PWD"
      NOMAD_RABBITMQ_HOST: rabbitmq
      NOMAD_ELASTIC_HOST: elastic
      NOMAD_MONGO_HOST: mongo
    env_file:
      - ./.env
    volumes:
      - ./configs/nomad.yaml:/app/nomad.yaml
      - /nomad/.volumes/fs:/app/.volumes/fs
      - /var/run/docker.sock:/var/run/docker.sock
    user: "1000:991"
    command: python -m nomad.cli admin run hub
    healthcheck:
      test:
        - "CMD"
        - "curl"
        - "--fail"
        - "--silent"
        - "http://localhost:8081/nomad-oasis/north/hub/health"
      interval: 10s
      timeout: 10s
      retries: 30
      start_period: 10s

  # nomad logtransfer
  # to enable the logtransfer service run "docker compose --profile with_logtransfer up"
  logtransfer:
    restart: unless-stopped
    image: ghcr.io/exciting/sol-oasis:main
    platform: linux/amd64
    container_name: nomad_oasis_logtransfer
    environment:
      NOMAD_SERVICE: nomad_oasis_logtransfer
      NOMAD_ELASTIC_HOST: elastic
      NOMAD_MONGO_HOST: mongo
    env_file:
      - ./.env
    depends_on:
      elastic:
        condition: service_healthy
      mongo:
        condition: service_healthy
    volumes:
      - ./configs/nomad.yaml:/app/nomad.yaml
      - ./.volumes/fs:/app/.volumes/fs
    command: python -m nomad.cli admin run logtransfer
    profiles: ["with_logtransfer"]

  # nomad proxy (a reverse proxy for nomad)
  proxy:
    restart: unless-stopped
    image: docker.io/nginx:1.27.4-alpine
    container_name: nomad_oasis_proxy
    command: nginx -g 'daemon off;'
    volumes:
      - ./configs/nginx.ssl.conf:/etc/nginx/conf.d/default.conf:ro
      - ./configs/nginx.full.conf:/etc/nginx/nginx.conf:ro
      - ${CERT_LOCAL_PATH}:/etc/nginx/ssl/:ro

    depends_on:
      app:
        condition: service_healthy
      worker:
        condition: service_started # TODO: service_healthy
      north:
        condition: service_healthy
    ports:
      - 80:80
      - 443:443
      - 27010:27010

  # jupyterlab image: (this is only used to pull the image, no service is run for this)
  jupyterlab_image:
    image: ghcr.io/exciting/sol-oasis/jupyter:main
    platform: linux/amd64
    container_name: nomad_jupyter_image
    pull_policy: always
    entrypoint: ["true"]
    deploy:
      replicas: 0 # Ensures the service isn't started

  jobflow_db:
    image: mongo:8.0
    container_name: jobflow_mongo
    restart: unless-stopped
    environment:
      - MONGO_DATA_DIR=/data/db
    env_file: .env
    command: mongod --logpath=/dev/null --auth --wiredTigerCacheSizeGB 4 #quiet, require authentification, limit cache size to 4 GB
    volumes:
      - /jobflow_db/jobflow_mongo:/data/db
      - /jobflow_db/jobflow_mongo_backup:/backup
      - ./configs/mongo-init.js:/docker-entrypoint-initdb.d/mongo-init.js:ro
      - ./configs/jobflow-users.js:/jobflow/jobflow-users.js

volumes:
  mongo:
    name: "nomad_oasis_mongo"
  mongo_config:
    name: "nomad_oasis_mongo_config"
  elastic:
    name: "nomad_oasis_elastic"
  rabbitmq:
    name: "nomad_oasis_rabbitmq"
  keycloak:
    name: "nomad_oasis_keycloak"
  postgresql:
    name: "nomad_oasis_postgresql"
networks:
  default:
    name: nomad_oasis_network
    driver_opts: 
      # adapt MTU for HU network
      com.docker.network.driver.mtu: 1396