docker-compose.yml

services:
    broker:
        image: rabbitmq:3-management-alpine
        restart: unless-stopped
        volumes:
            - broker-data:/var/lib/rabbitmq
        networks:
            - warehouse-backend
        environment:
            RABBITMQ_DEFAULT_USER: "${RABBITMQ_ADMIN}"
            RABBITMQ_DEFAULT_PASS: "${RABBITMQ_PW}"
        healthcheck:
            test: [ "CMD", "rabbitmq-diagnostics", "ping" ]
            interval: 10s
            timeout: 5s
            retries: 3

    flower:
        image: mher/flower
        #build:
        #dockerfile: ./docker/transform/Dockerfile
        #context: .
        networks:
            - warehouse-backend
        depends_on:
            broker:
                condition: service_healthy
            celery:
                condition: service_healthy
        volumes:
            #- ./data/results:/var/celery/results
            - flower-data:/data # persist flower db
        environment:
            CELERY_BROKER_URL: amqp://${RABBITMQ_ADMIN}:${RABBITMQ_PW}@broker:5672//
        command:
          [
              "celery",
              "flower",
              "--address=0.0.0.0",
              "--loglevel=info",
              "--persistent=True",
              "--db=flower_db"
#              "--port=5555"
            #"--without-heartbeat"
          ]

    celery:
        build:
            dockerfile: ./docker/transform/Dockerfile
            context: .
        volumes:
            - ./src:/code/src
            - ./config:/code/config
        networks:
            - warehouse-backend
        depends_on:
            broker:
                condition: service_healthy
        command:
          [
              "celery",
              "-A",
              "tasks",
              "worker",
            #"-Q",
            #"batches",
              "-E",
              "--pool=solo", # https://stackoverflow.com/questions/56767461/celery-workerlosterror-worker-exited-prematurely-signal-6-sigabrt
              "--loglevel=INFO"
          ]
        #volumes:
        #- ./data/results:/var/celery/results
        environment:
            PYTHONPATH: /code/src:/code  # src for tasks.py, root for config/
            CELERY_BROKER_URL: amqp://${RABBITMQ_ADMIN}:${RABBITMQ_PW}@broker:5672//
            EMBEDDING_MODEL: "${EMBEDDING_MODEL}"
            POSTGRES_USER: "${POSTGRES_ADMIN}"
            POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
            POSTGRES_DB: "${POSTGRES_DB}"
            FILE_DB: "${FILE_DB}"
            POSTGRES_ADDRESS: "${POSTGRES_ADDRESS}"
            POSTGRES_PORT: "${POSTGRES_PORT}"
            OPENSEARCH_ADDRESS: "${OPENSEARCH_ADDRESS}"
            OPENSEARCH_PORT: "${OPENSEARCH_PORT}"
        healthcheck:
            test: celery -A tasks status
            interval: 10s
            timeout: 5s
            retries: 3

    transform:
        build:
            dockerfile: ./docker/transform/Dockerfile
            context: .
        volumes:
            - ./src:/code/src
            - ./config:/code/config
        command: [ "fastapi", "run", "--host", "0.0.0.0", "transform.py", "--port", "80" ]
        networks:
            - warehouse-backend
        environment:
            PYTHONPATH: /code/src:/code
            CELERY_BROKER_URL: amqp://${RABBITMQ_ADMIN}:${RABBITMQ_PW}@broker:5672//
            POSTGRES_USER: "${POSTGRES_ADMIN}"
            POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
            POSTGRES_DB: "${POSTGRES_DB}"
            POSTGRES_ADDRESS: "${POSTGRES_ADDRESS}"
            POSTGRES_PORT: "${POSTGRES_PORT}"
            OPENSEARCH_ADDRESS: "${OPENSEARCH_ADDRESS}"
            OPENSEARCH_PORT: "${OPENSEARCH_PORT}"
            EMBEDDING_MODEL: "${EMBEDDING_MODEL}"
            CELERY_BATCH_SIZE: "${CELERY_BATCH_SIZE}"
        depends_on:
            postgres:
                condition: service_healthy
            opensearch:
                condition: service_healthy
            broker:
                condition: service_healthy

    postgres:
        image: postgres:17-alpine
        environment:
            POSTGRES_USER: "${POSTGRES_ADMIN}"
            POSTGRES_PASSWORD: "${POSTGRES_PASSWORD}"
            POSTGRES_DB: "${POSTGRES_DB}"
        restart: unless-stopped
        volumes:
            - postgres-data:/var/lib/postgresql/data
        networks:
            - warehouse-backend
        healthcheck:
            test: [ "CMD-SHELL", "pg_isready" ]
            interval: 10s
            timeout: 5s
            retries: 3


    pgadmin:
        image: dpage/pgadmin4:9.8
        container_name: pgadmin4_container
        restart: unless-stopped
        depends_on:
            postgres:
                condition: service_healthy
        environment:
            PGADMIN_DEFAULT_EMAIL: "${PGADMIN_ADMIN}"
            PGADMIN_DEFAULT_PASSWORD: "${PGADMIN_DEFAULT_PASSWORD}"
        volumes:
            - pgadmin-data:/var/lib/pgadmin
        networks:
            - warehouse-backend

    opensearch:
        restart: unless-stopped
        image: opensearchproject/opensearch:3.2.0
        networks:
            - warehouse-backend
        environment:
            discovery.type: single-node
            bootstrap.memory_lock: "true"
            OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
            cluster.name: opensearch-cluster
            plugins.security.disabled: "true"
            OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
        ulimits:
            memlock:
                soft: -1
                hard: -1
            nofile:
                soft: 65536
                hard: 65536
        volumes:
            - opensearch-data:/usr/share/opensearch/data
        healthcheck:
            test: [ "CMD-SHELL", "curl --silent --fail http://localhost:9200/_cluster/health || exit 1" ]

    opensearch-dashboards:
        restart: unless-stopped
        image: opensearchproject/opensearch-dashboards:3.2.0
        environment:
            - OPENSEARCH_HOSTS=["http://opensearch:9200"]
            - DISABLE_SECURITY_DASHBOARDS_PLUGIN=true
        volumes:
            - opensearch-dashboards-data:/usr/share/opensearch-dashboards/data
        depends_on:
            opensearch:
                condition: service_healthy
        networks:
            - warehouse-backend

    #https://github.com/EOSC-Data-Commons/matchmaker/pkgs/container/matchmaker-frontend
    frontend:
        image: ghcr.io/eosc-data-commons/matchmaker-frontend:latest
        restart: unless-stopped
        depends_on:
            opensearch:
                condition: service_healthy
        command: npm run prod
        environment:
            - NODE_ENV=production
            - SEARCH_API_URL=http://search-api:8000
            - PLAYER_API_URL=https://dev1.player.eosc-data-commons.eu
        networks:
            - warehouse-backend

    # https://github.com/EOSC-Data-Commons/data-commons-search/pkgs/container/data-commons-search
    search-api:
        restart: unless-stopped
        image: ghcr.io/eosc-data-commons/data-commons-search:latest
        depends_on:
            opensearch:
                condition: service_healthy
        volumes:
            - search-api-data:/app/data
        env_file: [ keys.env ]
        environment:
            - OPENSEARCH_URL=http://opensearch:9200
        networks:
            - warehouse-backend

    # https://github.com/EOSC-Data-Commons/metadata-crawlers/pkgs/container/metadata-crawlers
    harvester:
        image: ghcr.io/eosc-data-commons/metadata-crawlers:latest
        profiles: ["manual"]
        env_file:
            - .env
        volumes:
            - harvester-logs:/app/logs
        entrypoint: ["python", "-m", "harvester"]
        networks:
            - warehouse-backend

    scheduler:
      build:
        dockerfile: ./docker/scheduler/Dockerfile
        context: .
      profiles: ["manual"]
      command: ["python", "-m", "scheduler.run"]
      env_file:
        - .env
      depends_on:
        postgres:
          condition: service_healthy
        opensearch:
          condition: service_healthy
        transform:
          condition: service_started
      networks:
        - warehouse-backend
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
        - ./docker-compose.yml:/app/compose.yml
        - ./.env:/app/.env

volumes:
    postgres-data:
    pgadmin-data:
    opensearch-data:
    opensearch-dashboards-data:
    broker-data:
    flower-data:
    search-api-data:
    frontend-data:
    harvester-logs:

networks:
    warehouse-backend: