agentflow/docker-compose.flink.yml at main · brownjuly2003-code/agentflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
services:
  kafka:
    environment:
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,HOST://0.0.0.0:19092,CONTROLLER://0.0.0.0:29093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,HOST://localhost:19092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
    ports:
      - "9092:9092"
      - "19092:19092"
      - "9101:9101"

  flink-jobmanager:
    build:
      context: ./src
      dockerfile: processing/flink_jobs/Dockerfile
    image: agentflow-flink-local:latest
    command:
      - /bin/bash
      - -lc
      - |
        printf '%s\n' "$$FLINK_PROPERTIES" >> /opt/flink/conf/config.yaml
        printf '%s\n' 'jobmanager.memory.process.size: 1600m' >> /opt/flink/conf/config.yaml
        exec /opt/flink/bin/jobmanager.sh start-foreground
    environment:
      PYTHONPATH: /opt/agentflow

  flink-taskmanager:
    build:
      context: ./src
      dockerfile: processing/flink_jobs/Dockerfile
    image: agentflow-flink-local:latest
    command:
      - /bin/bash
      - -lc
      - |
        printf '%s\n' "$$FLINK_PROPERTIES" >> /opt/flink/conf/config.yaml
        printf '%s\n' 'taskmanager.memory.process.size: 1728m' >> /opt/flink/conf/config.yaml
        exec /opt/flink/bin/taskmanager.sh start-foreground
    environment:
      PYTHONPATH: /opt/agentflow

  flink-job-runner:
    build:
      context: ./src
      dockerfile: processing/flink_jobs/Dockerfile
    image: agentflow-flink-local:latest
    depends_on:
      kafka-init:
        condition: service_completed_successfully
      minio-init:
        condition: service_completed_successfully
      flink-jobmanager:
        condition: service_healthy
      flink-taskmanager:
        condition: service_started
    working_dir: /opt/agentflow
    environment:
      KAFKA_BOOTSTRAP_SERVERS: kafka:9092
      FLINK_PARALLELISM: "2"
      PYTHONPATH: /opt/agentflow
    command:
      - /bin/bash
      - -lc
      - |
        set -euo pipefail
        until curl -fsS http://flink-jobmanager:8081/overview >/dev/null; do
          sleep 2
        done
        /opt/flink/bin/flink run -d -m flink-jobmanager:8081 -py /opt/agentflow/src/processing/flink_jobs/stream_processor.py
        echo "Submitted src/processing/flink_jobs/stream_processor.py to the local Flink cluster."
        tail -f /dev/null