agentflow/docker-compose.yml at main · brownjuly2003-code/agentflow · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
services:
  # ── Kafka (KRaft mode, no Zookeeper) ──────────────────────────
  kafka:
    image: confluentinc/cp-kafka:7.7.0
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_LOG_RETENTION_HOURS: 24
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
      CLUSTER_ID: "AgentFlowKafkaCluster01"
      KAFKA_JMX_PORT: 9101
    volumes:
      - kafka-data:/var/lib/kafka/data
    healthcheck:
      test: kafka-broker-api-versions --bootstrap-server localhost:9092
      interval: 10s
      timeout: 10s
      retries: 5

  kafka-init:
    image: confluentinc/cp-kafka:7.7.0
    depends_on:
      kafka:
        condition: service_healthy
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        echo "Creating Kafka topics..."
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic orders.raw --partitions 6 --replication-factor 1
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic payments.raw --partitions 6 --replication-factor 1
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic clicks.raw --partitions 6 --replication-factor 1
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic products.cdc --partitions 3 --replication-factor 1
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic events.validated --partitions 6 --replication-factor 1
        kafka-topics --bootstrap-server kafka:9092 --create --if-not-exists --topic events.deadletter --partitions 3 --replication-factor 1
        echo "Topics created."

  # ── Flink ─────────────────────────────────────────────────────
  flink-jobmanager:
    image: flink:2.2.1-java17
    ports:
      - "8081:8081"
    command: jobmanager
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        jobmanager.bind-host: 0.0.0.0
        rest.bind-host: 0.0.0.0
        state.backend.type: rocksdb
        execution.checkpointing.dir: s3://agentflow-lake/checkpoints
        s3.endpoint: http://minio:9000
        s3.access-key: minio
        s3.secret-key: minio123
        s3.path.style.access: true
        execution.checkpointing.interval: 30s
        execution.checkpointing.mode: EXACTLY_ONCE
    volumes:
      - flink-checkpoints:/opt/flink/checkpoints
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8081/overview"]
      interval: 10s
      timeout: 5s
      retries: 5

  flink-taskmanager:
    image: flink:2.2.1-java17
    depends_on:
      flink-jobmanager:
        condition: service_healthy
    command: taskmanager
    environment:
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.bind-host: 0.0.0.0
        taskmanager.numberOfTaskSlots: 4
        state.backend.type: rocksdb
        s3.endpoint: http://minio:9000
        s3.access-key: minio
        s3.secret-key: minio123
        s3.path.style.access: true
    deploy:
      replicas: 2

  # ── MinIO (S3-compatible object store) ────────────────────────
  minio:
    image: minio/minio:RELEASE.2025-09-07T16-13-09Z
    ports:
      - "9000:9000"
      - "9001:9001"
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: minio
      MINIO_ROOT_PASSWORD: minio123
    volumes:
      - minio-data:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 3

  minio-init:
    image: minio/mc:RELEASE.2025-08-13T08-35-41Z
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        mc alias set local http://minio:9000 minio minio123
        mc mb local/agentflow-lake --ignore-existing
        mc mb local/agentflow-lake/warehouse --ignore-existing
        mc mb local/agentflow-lake/checkpoints --ignore-existing
        echo "MinIO buckets created."

  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 3

  # ── Prometheus ────────────────────────────────────────────────
  prometheus:
    image: prom/prometheus:v2.54.0
    ports:
      - "9090:9090"
    volumes:
      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./monitoring/alerting/rules.yml:/etc/prometheus/rules.yml
      - prometheus-data:/prometheus

  # ── Grafana ───────────────────────────────────────────────────
  grafana:
    image: grafana/grafana:11.2.0
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/pipeline_health.json
    volumes:
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
      - grafana-data:/var/lib/grafana
    depends_on:
      - prometheus

volumes:
  kafka-data:
  flink-checkpoints:
  minio-data:
  prometheus-data:
  grafana-data: