From a7d8c7f5a1a1018594ec290762021e619a8c552f Mon Sep 17 00:00:00 2001 From: m0533199321 Date: Wed, 12 Nov 2025 19:44:41 +0200 Subject: [PATCH 1/2] change the network in docker-compose --- docker-compose.yml | 726 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 550 insertions(+), 176 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index da66a514d..61799b947 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,17 +1,16 @@ - # ========================== # Docker Compose - AG Cloud # ========================== -version: "3.9" +# version: "3.9" # -------------------------- # Networks # -------------------------- networks: ag_cloud: + name: ag_cloud driver: bridge - # -------------------------- # Volumes # -------------------------- @@ -53,7 +52,7 @@ services: - wal_archive:/var/lib/postgresql/wal_archive - backups:/var/lib/postgresql/backups healthcheck: - test: ["CMD", "pg_isready", "-U", "missions_user", "-d", "missions_db"] + test: [ "CMD", "pg_isready", "-U", "missions_user", "-d", "missions_db" ] interval: 10s timeout: 5s retries: 5 @@ -76,34 +75,91 @@ services: - "9187:9187" networks: - ag_cloud + # ------------------------- + # Sound Metrics Service + # ------------------------- - plant_stress: - build: ./services/plant_stress + sound_metrics: + build: + context: ./services/sound_metrics + dockerfile: Dockerfile environment: - - INPUT_DIR=/data/inbox - - MODEL_DIR=/models - - POSTGRES_DSN=postgresql://missions_user:pg123@postgres:5432/missions_db - - PERIOD_DAYS=0 - - CONFIDENCE_THRESHOLD=0.6 - # - TF_ENABLE_ONEDNN_OPTS=0 # optional - volumes: - - "./services/plant_stress/models:/models:ro" - - "./services/plant_stress/samples:/data/inbox:ro" + - ADDR=0.0.0.0 + - PORT=8005 + - USE_UTC=false + - WINDOW_MIN=1 + - STABLE_SEC=1 + - PYTHONUNBUFFERED=1 + - MINIO_ENDPOINT=minio-hot:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin123 + - MINIO_BUCKET=sound + - MINIO_PREFIX=sounds/ + + command: [ "python", "-u", "src/metrics.py" ] + ports: + - "8005:8005" depends_on: - - postgres - command: ["python", "-u", "/app/app.py"] + - minio-hot networks: - ag_cloud + restart: unless-stopped - + # ------------------------- + # Plant Stress Daily Batch + # ------------------------- + + plant_stress_daily: + build: ./services/plant_stress + env_file: + - ./services/plant_stress/.env + restart: "no" + environment: + MODEL_DIR: /models + CONFIDENCE_THRESHOLD: "0.60" + TF_CPP_MIN_LOG_LEVEL: "2" + TIMEZONE: Asia/Jerusalem + POSTGRES_DSN: postgresql://missions_user:pg123@postgres:5432/missions_db + MINIO_ENDPOINT: minio-hot:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin123 + MINIO_BUCKET: sound + MINIO_PREFIX: plants/ + MINIO_SECURE: "false" + + DEFAULT_AREA: unknown + DEFAULT_LAT: 0.0 + DEFAULT_LON: 0.0 + DEFAULT_IMAGE_URL: https://example.com/placeholder.jpg + DEFAULT_VOD: https://example.com/placeholder.mp4 + DEFAULT_HLS: https://example.com/placeholder.m3u8 + # ==== Alerts → Kafka ==== + ENABLE_ALERTS: "true" + KAFKA_BOOTSTRAP: "kafka:9092" + ALERT_TOPIC: "alerts" + ALERT_TYPE: "plant_drought_detected" + KAFKA_CLIENT_ID: "plant-stress-producer" + command: ["python","-u","/app/predict_minio_daily.py"] + volumes: + - "./services/plant_stress/models:/models:ro" + depends_on: + postgres: + condition: service_healthy + minio-hot: + condition: service_healthy + mc-bootstrap: + condition: service_started + kafka: + condition: service_healthy + networks: [ag_cloud] # ------------------------- - # MQTT + Kafka + Connect + Init + # MQTT + Kafka + MQTT-router # ------------------------- kafka: - build: + build: context: ./mqtt_and_kafka/kafka dockerfile: dockerfile container_name: kafka @@ -128,7 +184,7 @@ services: networks: - ag_cloud healthcheck: - test: ["CMD-SHELL", "/opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list >/dev/null 2>&1 || exit 1"] + test: [ "CMD-SHELL", "/opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 --list >/dev/null 2>&1 || exit 1" ] interval: 10s timeout: 5s retries: 20 @@ -136,7 +192,7 @@ services: mosquitto: image: eclipse-mosquitto:2.0 container_name: mosquitto - command: ["mosquitto", "-c", "/mqtt_and_kafka/mosquitto/config/mosquitto.conf"] + command: [ "mosquitto", "-c", "/mqtt_and_kafka/mosquitto/config/mosquitto.conf" ] ports: - "1883:1883" volumes: @@ -147,60 +203,37 @@ services: networks: - ag_cloud healthcheck: - test: ["CMD", "mosquitto_sub", "-h", "localhost", "-p", "1883", "-t", "$$SYS/#", "-C", "1", "-W", "15"] + test: [ "CMD", "mosquitto_sub", "-h", "localhost", "-p", "1883", "-t", "$$SYS/#", "-C", "1", "-W", "15" ] interval: 10s timeout: 5s retries: 12 - connect: - build: - context: ./mqtt_and_kafka - dockerfile: connect.Dockerfile - image: local/connect-with-mqtt:1.0.0 - container_name: connect - depends_on: - kafka: - condition: service_healthy - mosquitto: - condition: service_healthy - ports: - - "8083:8083" - environment: - - CONNECT_BOOTSTRAP_SERVERS=kafka:9092 - - CONNECT_GROUP_ID=agcloud-connect - - CONNECT_CONFIG_STORAGE_TOPIC=_connect_configs - - CONNECT_OFFSET_STORAGE_TOPIC=_connect_offsets - - CONNECT_STATUS_STORAGE_TOPIC=_connect_status - - CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR=1 - - CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR=1 - - CONNECT_STATUS_STORAGE_REPLICATION_FACTOR=1 - - CONNECT_KEY_CONVERTER=org.apache.kafka.connect.storage.StringConverter - - CONNECT_VALUE_CONVERTER=org.apache.kafka.connect.storage.StringConverter - - CONNECT_REST_ADVERTISED_HOST_NAME=localhost - - CONNECT_PLUGIN_PATH=/usr/share/java,/usr/share/confluent-hub-components - networks: - - ag_cloud - healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:8083/connectors"] - interval: 10s - timeout: 5s - retries: 12 + mqtt-router: + build: + context: ./mqtt_and_kafka/mqtt-router + image: local/mqtt-router:1.0.0 + depends_on: + kafka: + condition: service_healthy + mosquitto: + condition: service_healthy + environment: + - MQTT_HOST=mosquitto + - MQTT_PORT=1883 + - MQTT_TOPIC_FILTER=mqtt/# - init-connector: - image: curlimages/curl:8.7.1 - depends_on: - connect: - condition: service_healthy - volumes: - - ./mqtt_and_kafka/connectors:/connectors - networks: - - ag_cloud - entrypoint: > - sh -c " - echo '==> Creating MQTT connector...'; - curl -X POST -H 'Content-Type: application/json' --data @/connectors/mqtt-source.json http://connect:8083/connectors; - echo '==> Done.'; - " + - KAFKA_BOOTSTRAP=kafka:9092 + - CREATE_TOPICS=false + - DEFAULT_PARTITIONS=1 + - DEFAULT_REPLICATION=1 + networks: + - ag_cloud + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import socket; socket.create_connection(('mosquitto',1883),3); socket.create_connection(('kafka',9092),3)"] + interval: 15s + timeout: 5s + retries: 5 # -------------------------- # GUI / Runner / Gateway @@ -290,6 +323,15 @@ services: networks: - ag_cloud + pushgateway: + image: prom/pushgateway:v1.8.0 + container_name: pushgateway + ports: + - "9091:9091" + networks: + - ag_cloud + restart: unless-stopped + # -------------------------- # Desktop App # -------------------------- @@ -303,6 +345,10 @@ services: - DISPLAY=host.docker.internal:0.0 - GATEWAY_URL=http://sensors_metrics:8000 - NOTIFICATION_API_URL=http://notification_api:5000 + + - API_BASE_URL=http://db_api_service:8001 + - AUTH_BOOTSTRAP_URL=http://db_api_service:8001/auth/_dev_bootstrap + - ALERTS_WS_URL=ws://alerts-gateway:8000/ws/alerts ports: - "5900:5900" - "8080:8080" @@ -311,13 +357,12 @@ services: - notification_api - alerts-gateway volumes: - - ./GUI/src/vast:/app/src/vast - - ./templates:/app/templates:ro + - ./GUI/src/vast:/app/src/vast + - ./templates:/app/templates:ro networks: - ag_cloud restart: unless-stopped - # -------------------------- # Large Mosquitto # -------------------------- @@ -343,12 +388,51 @@ services: MINIO_PROMETHEUS_AUTH_TYPE: public MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin123 + + # ===== IMAGE NOTIFIERS ===== + MINIO_NOTIFY_KAFKA_ENABLE_aerial: "on" + MINIO_NOTIFY_KAFKA_BROKERS_aerial: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_aerial: "image.new.aerial" + + MINIO_NOTIFY_KAFKA_ENABLE_air: "on" + MINIO_NOTIFY_KAFKA_BROKERS_air: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_air: "image.new.air" + + MINIO_NOTIFY_KAFKA_ENABLE_fruits: "on" + MINIO_NOTIFY_KAFKA_BROKERS_fruits: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_fruits: "image.new.fruits" + + MINIO_NOTIFY_KAFKA_ENABLE_leaves: "on" + MINIO_NOTIFY_KAFKA_BROKERS_leaves: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_leaves: "image.new.leaves" + + MINIO_NOTIFY_KAFKA_ENABLE_ground: "on" + MINIO_NOTIFY_KAFKA_BROKERS_ground: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_ground: "image.new.ground" + + MINIO_NOTIFY_KAFKA_ENABLE_field: "on" + MINIO_NOTIFY_KAFKA_BROKERS_field: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_field: "image.new.field" + + # ===== SOUND NOTIFIERS ===== + MINIO_NOTIFY_KAFKA_ENABLE_plants: "on" + MINIO_NOTIFY_KAFKA_BROKERS_plants: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_plants: "sound.new.plants" + + MINIO_NOTIFY_KAFKA_ENABLE_sounds: "on" + MINIO_NOTIFY_KAFKA_BROKERS_sounds: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_sounds: "sound.new.sounds" + + # ===== SECURITY NOTIFIER ===== + MINIO_NOTIFY_KAFKA_ENABLE_security: "on" + MINIO_NOTIFY_KAFKA_BROKERS_security: "kafka:9092" + MINIO_NOTIFY_KAFKA_TOPIC_security: "image.new.security" ports: - - "9001:9000" # HOT S3 - - "9002:9001" # HOT Console - networks: [ag_cloud] + - "9001:9000" # HOT S3 + - "9002:9001" # HOT Console + networks: [ ag_cloud ] healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:9000/minio/health/ready"] + test: [ "CMD", "curl", "-fsS", "http://localhost:9000/minio/health/ready" ] interval: 3s timeout: 2s retries: 40 @@ -364,11 +448,11 @@ services: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin123 ports: - - "9101:9000" # COLD S3 - - "9102:9001" # COLD Console - networks: [ag_cloud] + - "9101:9000" # COLD S3 + - "9102:9001" # COLD Console + networks: [ ag_cloud ] healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:9000/minio/health/ready"] + test: [ "CMD", "curl", "-fsS", "http://localhost:9000/minio/health/ready" ] interval: 3s timeout: 2s retries: 40 @@ -378,6 +462,7 @@ services: mc-bootstrap: build: context: ./storage_with_mqtt/storage/Lifecycle_rules/minio-bootstrap + container_name: mc-bootstrap volumes: - ./storage_with_mqtt/storage/combined_minio_setup/config:/config:ro - ./storage_with_mqtt/data/config:/config @@ -386,7 +471,8 @@ services: condition: service_healthy minio-cold: condition: service_healthy - command: ["/bin/bash","-lc","/entrypoint/init.sh; tail -f /dev/null"] + kafka: + condition: service_healthy environment: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin123 @@ -395,8 +481,8 @@ services: MC_ALIAS_HOT: hot MC_ALIAS_COLD: cold BUCKET_IMAGERY: imagery - BUCKET_TELEMETRY: telemetry - networks: [ag_cloud] + BUCKET_SOUND: sound + networks: [ ag_cloud ] restart: unless-stopped # -------------------------- @@ -411,7 +497,7 @@ services: MINIO_ROOT_USER: minioadmin MINIO_ROOT_PASSWORD: minioadmin123 BUCKET_IMAGERY: imagery - BUCKET_TELEMETRY: telemetry + BUCKET_SOUND: sound MQTT_BROKER: large-mosquitto MQTT_PORT: 1885 MQTT_TOPIC: MQTT/imagery/# @@ -438,6 +524,83 @@ services: - ag_cloud restart: unless-stopped + mqtt_ingest_sound: + build: + context: ./storage_with_mqtt/mqtt_images/mqtt_ingest + container_name: mqtt_ingest_sound + environment: + MINIO_ENDPOINT: http://minio-hot:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin123 + S3_BUCKET: sound + MQTT_BROKER: large-mosquitto + MQTT_PORT: 1885 + MQTT_TOPIC: MQTT/sounds/# + MQTT_PUB_TOPIC: sound/sounds/ingested + DEFAULT_PREFIX: MIC-01 + CAMERA_PREFIX: camera + MICROPHONE_PREFIX: microphone + DUMMY_DB: 0 + DB_API_BASE: http://db_api_service:8001 + DB_API_TOKEN: auto + OUTBOX_DIR: /app/outbox + DB_API_AUTH_MODE: service + DB_API_SERVICE_NAME: mqtt_ingest_sound + INGEST_WORKERS: 8 + volumes: + - ./storage_with_mqtt/mqtt_images/outbox:/app/outbox + depends_on: + large-mosquitto: + condition: service_started + minio-hot: + condition: service_healthy + mc-bootstrap: + condition: service_started + db_api_service: + condition: service_started + networks: + - ag_cloud + restart: unless-stopped + + mqtt_ingest_sounds_ultra: + build: + context: ./storage_with_mqtt/mqtt_images/mqtt_ingest + container_name: mqtt_ingest_sounds_ultra + environment: + MINIO_ENDPOINT: http://minio-hot:9000 + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin123 + S3_BUCKET: sound + MQTT_BROKER: large-mosquitto + MQTT_PORT: 1885 + MQTT_TOPIC: MQTT/sounds_ultra/# + MQTT_PUB_TOPIC: sound/sounds_ultra/ingested + DEFAULT_PREFIX: MIC-02 + CAMERA_PREFIX: microphone + MICROPHONE_PREFIX: microphone + DUMMY_DB: 0 + DB_API_BASE: http://db_api_service:8001 + DB_API_TOKEN: auto + OUTBOX_DIR: /app/outbox + DB_API_AUTH_MODE: service + DB_API_SERVICE_NAME: mqtt_ingest_sounds_ultra + INGEST_WORKERS: 8 + ULTRA_DIR_PREFIX: plants + volumes: + - ./storage_with_mqtt/mqtt_images/outbox:/app/outbox + depends_on: + large-mosquitto: + condition: service_started + minio-hot: + condition: service_healthy + mc-bootstrap: + condition: service_started + db_api_service: + condition: service_started + networks: + - ag_cloud + restart: unless-stopped + mqtt_publisher: build: context: ./storage_with_mqtt/mqtt_images/mqtt_publisher @@ -465,14 +628,10 @@ services: # ------------------------ sounds_classifier: build: - context: ./services/sounds/sounds_classifier + context: ./services/sounds_classifier dockerfile: Dockerfile.classifier-svc - # args: - # CHECKPOINT_URL: "CHECKPOINT=/app/classification/models/panns_data/Cnn14_mAP=0.431.pth" container_name: sounds_classifier restart: unless-stopped - # env_file: - # - ./services/sounds/sounds_classifier/src/classification/.env environment: # Runtime mode - DEVICE=cpu @@ -491,7 +650,8 @@ services: # Kafka - KAFKA_BROKERS=kafka:9092 - - ALERTS_TOPIC=dev-robot-alerts + - ALERTS_TOPIC=alerts + - ENABLE_ALERTS=true # MinIO - MINIO_ENDPOINT=minio-hot:9000 @@ -500,7 +660,7 @@ services: - MINIO_SECURE=false # Request validation - - ALLOWED_BUCKETS=imagery + - ALLOWED_BUCKETS=sound - ALLOWED_CONTENT_TYPES=audio/wav,audio/x-wav,audio/mpeg,audio/flac,audio/ogg,audio/mp4 - MAX_BYTES=104857600 @@ -522,7 +682,7 @@ services: networks: - ag_cloud healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8088/health"] + test: [ "CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8088/health').read()" ] interval: 45s timeout: 5s retries: 10 @@ -532,8 +692,6 @@ services: # DB API Service # -------------------------- - - contracts-gen: build: context: ./services/db_api_service @@ -551,7 +709,6 @@ services: - ag_cloud restart: "no" - db_api_service: build: context: ./services/db_api_service @@ -560,16 +717,18 @@ services: env_file: - ./services/db_api_service/.env environment: - DB_DSN: postgresql+psycopg://missions_user:pg123@host.docker.internal:5432/missions_db + DB_DSN: postgresql+psycopg://missions_user:pg123@postgres:5432/missions_db ENV: dev JWT_SECRET: change-me-please-very-secret JWT_ALGO: HS256 ACCESS_TTL_MIN: 15 REFRESH_TTL_DAYS: 14 DEV_SA_NAME: my-ingest-service + ADDR: 0.0.0.0 ports: - "8001:8001" volumes: + - ./services/db_api_service/app:/app/app - contracts:/app/app/contracts depends_on: contracts-gen: @@ -580,7 +739,6 @@ services: - ag_cloud restart: unless-stopped - notification_api: build: context: ./services/API-notifications/src @@ -592,9 +750,40 @@ services: - "5000:5000" depends_on: - postgres + networks: + - ag_cloud - - + ripeness-api: + build: + context: ./services/ripeness-ml + dockerfile: deploy/Dockerfile + image: ripeness-api:latest + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGDATABASE=missions_db + - PGUSER=missions_user + - PGPASSWORD=pg123 + - MINIO_ENDPOINT=minio-hot:9000 + - MINIO_SECURE=false + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin123 + - MODEL_NAME=best_conditional + - BATCH_LIMIT=500 + - FRUITS=Apple,Banana,Orange + depends_on: + - postgres + - minio-hot + volumes: + - ./services/ripeness-ml/checkpoints:/app/checkpoints + - ./services/ripeness-ml/configs:/app/configs + - ./services/ripeness-ml/model:/app/model + container_name: ripeness-api + networks: [ ag_cloud ] + ports: + - "8091:8088" + restart: unless-stopped + # -------------------------- # Flink JobManager & TaskManager # -------------------------- @@ -602,12 +791,12 @@ services: build: context: ./streaming/flink dockerfile: Dockerfile.flink-py - image: agcloud-flink-py:1.18 + image: agcloud-flink-py:1.18 container_name: flink-jobmanager command: jobmanager ports: - "8081:8081" - networks: [ag_cloud] + networks: [ ag_cloud ] environment: - | FLINK_PROPERTIES= @@ -623,7 +812,7 @@ services: fs.s3a.connection.ssl.enabled: false python.client.executable: /usr/bin/python3 python.executable: /usr/bin/python3 - - HTTP_INFER_URL=http://fruit-inference-http:8000/infer_json + - HTTP_INFER_URL=http://fruit-inference-http:8004/infer_json volumes: - ./streaming/flink/jobs:/opt/flink/jobs:ro - ./streaming/flink/connectors/flink-json-1.18.1.jar:/opt/flink/lib/flink-json-1.18.1.jar:ro @@ -633,29 +822,29 @@ services: - ./streaming/flink/connectors/lz4-java-1.8.0.jar:/opt/flink/lib/lz4-java-1.8.0.jar:ro - ./streaming/flink/connectors/snappy-java-1.1.10.5.jar:/opt/flink/lib/snappy-java-1.1.10.5.jar:ro restart: unless-stopped - + audio_compression: - build: - context: ./services/compression - dockerfile: Dockerfile - container_name: audio_compression - environment: - - RAW_MAX_AGE_DAYS=30 - - COMPRESSION_CODEC=opus - - COMPRESSED_MAX_AGE_DAYS=90 - - CHECK_INTERVAL_SECONDS=3600 - - MINIO_ENDPOINT=minio-hot:9000 - - ACCESS_KEY=minioadmin - - SECRET_KEY=minioadmin123 - - BUCKET_NAME=imagery - depends_on: - minio-hot: - condition: service_healthy - mc-bootstrap: - condition: service_started - networks: - - ag_cloud - restart: unless-stopped + build: + context: ./services/compression + dockerfile: Dockerfile + container_name: audio_compression + environment: + - RAW_MAX_AGE_DAYS=30 + - COMPRESSION_CODEC=opus + - COMPRESSED_MAX_AGE_DAYS=90 + - CHECK_INTERVAL_SECONDS=3600 + - MINIO_ENDPOINT=minio-hot:9000 + - ACCESS_KEY=minioadmin + - SECRET_KEY=minioadmin123 + - BUCKET_NAME=imagery + depends_on: + minio-hot: + condition: service_healthy + mc-bootstrap: + condition: service_started + networks: + - ag_cloud + restart: unless-stopped flink_writer_db: build: @@ -664,7 +853,7 @@ services: container_name: flink_writer_db environment: - KAFKA_BROKERS=kafka:9092 - - TOPICS=sensor_zone_stats,sensor_anomalies,aerial_images_keys,aerial_image_object_detections,aerial_image_anomaly_detections + - TOPICS=sensor_zone_stats,sensor_anomalies,image_new_security_connections,alerts,image_new_aerial_connections,aerial_images_metadata,aerial_image_object_detections,aerial_image_anomaly_detections,aerial_images_complete_metadata,aerial_image_segmentation,sound_new_sounds_connections,sound_new_plants_connections,sounds_metadata,sounds_ultra_metadata - DB_API_BASE=http://db_api_service:8001 - DB_API_AUTH_MODE=service - DB_API_SERVICE_NAME=flink-writer-db @@ -679,7 +868,6 @@ services: - ag_cloud restart: unless-stopped - flink-taskmanager: image: agcloud-flink-py:1.18 container_name: flink-taskmanager @@ -687,7 +875,7 @@ services: depends_on: flink-jobmanager: condition: service_started - networks: [ag_cloud] + networks: [ ag_cloud ] environment: - | FLINK_PROPERTIES= @@ -703,7 +891,7 @@ services: fs.s3a.connection.ssl.enabled: false python.client.executable: /usr/bin/python3 python.executable: /usr/bin/python3 - - HTTP_INFER_URL=http://fruit-inference-http:8000/infer_json + - HTTP_INFER_URL=http://fruit-inference-http:8004/infer_json volumes: - ./streaming/flink/connectors/flink-json-1.18.1.jar:/opt/flink/lib/flink-json-1.18.1.jar:ro - ./streaming/flink/connectors/flink-sql-connector-kafka-3.2.0-1.18.jar:/opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar:ro @@ -713,31 +901,76 @@ services: - ./streaming/flink/connectors/snappy-java-1.1.10.5.jar:/opt/flink/lib/snappy-java-1.1.10.5.jar:ro restart: unless-stopped - - # -------------------------- # Inference HTTP Service # -------------------------- fruit-inference-http: build: - context: ./services/inference_http + context: ./services/inference_http dockerfile: Dockerfile environment: - - TEAM=fruit - - WEIGHTS_PATH=/app/weights/fruit_cls_best.ts - - MINIO_ENDPOINT=minio-hot:9000 - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin123 - - MINIO_SECURE=0 + - TEAM=fruit + - WEIGHTS_PATH=/app/weights/fruit_cls_best.ts + - MINIO_ENDPOINT=minio-hot:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin123 + - MINIO_SECURE=0 volumes: - ./services/inference_http/weights:/app/weights:ro container_name: fruit-inference-http + networks: [ ag_cloud ] + ports: + - "8011:8004" + restart: unless-stopped + + camera-inference-http: + build: + context: ./services/inference_http + dockerfile: Dockerfile + environment: + - TEAM=camera + - WEIGHTS_PATH=/app/weights/yolov8-fruits.pt + - MINIO_ENDPOINT=minio-hot:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin123 + - MINIO_SECURE=0 + volumes: + - ./services/inference_http/weights:/app/weights:ro + container_name: camera-inference-http networks: [ag_cloud] ports: - - "8011:8000" + - "8012:8004" + restart: unless-stopped + soil-inference-http: + build: + context: ./services/inference_http + dockerfile: Dockerfile + environment: + + - TEAM=soil_moisture + - WEIGHTS_PATH=/app/weights/soil_moisture_best.onnx + - MINIO_ENDPOINT=minio-hot:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin123 + - MINIO_SECURE=0 + - PG_DSN=postgresql://missions_user:pg123@postgres:5432/missions_db + - KAFKA_BROKERS=kafka:9092 + - KAFKA_TOPIC=irrigation.control + - KAFKA_DLT=irrigation.control.dlq + + + volumes: + - ./services/inference_http/weights:/app/weights:ro + - ./services/inference_http/adapters:/app/adapters + - ./services/inference_http/soil_moisture:/app/soil_moisture + depends_on: + - minio-hot + - postgres + ports: + - "8013:8004" + networks: [ag_cloud] restart: unless-stopped - # -------------------------- # Flink Jobs # -------------------------- @@ -748,12 +981,12 @@ services: flink-jobmanager: { condition: service_started } flink-taskmanager: { condition: service_started } fruit-inference-http: { condition: service_started } - networks: [ag_cloud] + networks: [ ag_cloud ] environment: - KAFKA_BOOTSTRAP=kafka:9092 - INPUT_TOPIC=imagery.new.fruit - TEAM=fruit - - HTTP_URL=http://fruit-inference-http:8000/infer_json + - HTTP_URL=http://fruit-inference-http:8004/infer_json - DLQ_TOPIC=dlq.inference.http - GROUP_ID=http-dispatcher-fruit - PARALLELISM=2 @@ -766,31 +999,53 @@ services: - ./streaming/flink/connectors/kafka-clients-3.2.3.jar:/opt/flink/lib/kafka-clients-3.2.3.jar:ro - ./streaming/flink/connectors/lz4-java-1.8.0.jar:/opt/flink/lib/lz4-java-1.8.0.jar:ro - ./streaming/flink/connectors/snappy-java-1.1.10.5.jar:/opt/flink/lib/snappy-java-1.1.10.5.jar:ro - command: [ - "bash","-lc", - "set -e; - echo 'Waiting for JobManager to accept commands...'; - until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do - echo 'still waiting...'; sleep 3; - done; - echo 'JobManager is ready!'; - /opt/flink/bin/flink run \ - -Dpython.client.executable=/usr/bin/python3 \ - -Dpython.executable=/usr/bin/python3 \ - -Dpipeline.jars=file:///opt/flink/lib/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-json-1.18.1.jar \ - --jobmanager flink-jobmanager:8081 \ - --detached \ - --python /opt/flink/jobs/http_dispatcher.py \ - -- \ - --bootstrap kafka:9092 \ - --input-topic imagery.new.fruit \ - --team fruit \ - --http-url http://fruit-inference-http:8000/infer_json \ - --group-id http-dispatcher-fruit \ - --dlq-topic dlq.inference.http; - tail -f /dev/null" - ] + command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic imagery.new.fruit --team fruit --http-url http://fruit-inference-http:8004/infer_json --group-id http-dispatcher-fruit --dlq-topic dlq.inference.http; tail -f /dev/null" ] + restart: always + + flink-dispatcher-camera: + image: agcloud-flink-py:1.18 + container_name: flink-dispatcher-camera + depends_on: + flink-jobmanager: { condition: service_started } + flink-taskmanager: { condition: service_started } + camera-inference-http: { condition: service_started } + networks: [ag_cloud] + environment: + - KAFKA_BOOTSTRAP=kafka:9092 + - INPUT_TOPIC=imagery.new.camera + - TEAM=camera + - HTTP_URL=http://camera-inference-http:8004/infer_json + - DLQ_TOPIC=dlq.inference.http + - GROUP_ID=http-dispatcher-camera + - PARALLELISM=2 + - PYFLINK_CLIENT_EXECUTABLE=/usr/bin/python3 + volumes: + - ./streaming/flink/jobs:/opt/flink/jobs:ro + - ./streaming/flink/connectors:/opt/flink/lib/connectors:ro + command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/connectors/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic imagery.new.camera --team camera --http-url http://camera-inference-http:8004/infer_json --group-id http-dispatcher-camera --dlq-topic dlq.inference.http; tail -f /dev/null" ] + restart: always + flink-dispatcher-soil: + image: agcloud-flink-py:1.18 + depends_on: + flink-jobmanager: { condition: service_started } + flink-taskmanager: { condition: service_started } + soil-inference-http: { condition: service_started } + networks: [ag_cloud] + environment: + - KAFKA_BOOTSTRAP=kafka:9092 + - INPUT_TOPIC=image.new.ground + - TEAM=soil_moisture + - HTTP_URL=http://soil-inference-http:8004/infer_json + - DLQ_TOPIC=dlq.inference.http + - GROUP_ID=http-dispatcher-soil + - PARALLELISM=1 + - PYFLINK_CLIENT_EXECUTABLE=/usr/bin/python3 + volumes: + - ./streaming/flink/jobs:/opt/flink/jobs:ro + - ./streaming/flink/connectors:/opt/flink/lib/connectors:ro + command: [ "bash", "-lc", "set -e; echo 'Waiting...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/connectors/... --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic image.new.ground --team soil_moisture --http-url http://soil-inference-http:8004/infer_json --group-id http-dispatcher-soil --dlq-topic dlq.inference.http; tail -f /dev/null" ] + flink-alerts-job: build: @@ -799,18 +1054,17 @@ services: container_name: alerts-forwarder depends_on: kafka: - condition: service_healthy + condition: service_healthy alertmanager_service: - condition: service_started + condition: service_started environment: - PYTHONPATH=/opt/app - KAFKA_BROKERS=kafka:9092 - ALERTMANAGER_SERVICE_URL=http://alertmanager_service:8090/alerts - command: ["python", "/opt/app/alerts_forwarder.py"] + command: [ "python", "/opt/app/alerts_forwarder.py" ] networks: - ag_cloud restart: unless-stopped - alertmanager: image: prom/alertmanager:v0.27.0 @@ -826,7 +1080,7 @@ services: networks: - ag_cloud restart: always - + alertmanager_service: build: context: ./services/alertmanager_service/src @@ -834,7 +1088,7 @@ services: container_name: alertmanager_service ports: - "8090:8090" - command: ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8090"] + command: [ "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8090" ] volumes: - ./templates:/app/templates:ro environment: @@ -852,10 +1106,130 @@ services: context: ./services/alertmanager_service/src dockerfile: Dockerfile container_name: alerts_gateway - command: ["uvicorn", "gateway:app", "--host", "0.0.0.0", "--port", "8000"] + command: [ "uvicorn", "gateway:app", "--host", "0.0.0.0", "--port", "8000" ] ports: - - "8010:8000" # host:container + - "8010:8000" + networks: + - ag_cloud + + image-linker-jobmanager: + build: + context: ./services/image-linker + dockerfile: Dockerfile.flink + container_name: image-linker-jobmanager + command: jobmanager + ports: + - "8084:8081" + environment: + - JOB_MANAGER_RPC_ADDRESS=image-linker-jobmanager + - KAFKA_BROKERS=kafka:9092 + - CONFIG_PATH=/opt/app/config/topics.yaml networks: - ag_cloud - + image-linker-taskmanager: + build: + context: ./services/image-linker + dockerfile: Dockerfile.flink + container_name: image-linker-taskmanager + command: taskmanager + environment: + - JOB_MANAGER_RPC_ADDRESS=image-linker-jobmanager + - KAFKA_BROKERS=kafka:9092 + - CONFIG_PATH=/opt/app/config/topics.yaml + depends_on: + image-linker-jobmanager: + condition: service_started + networks: + - ag_cloud + + image-linker-submitter: + build: + context: ./services/image-linker + dockerfile: Dockerfile.flink + container_name: image-linker-submit + depends_on: + image-linker-jobmanager: + condition: service_started + command: > + bash -lc "sleep 10 && + flink run -m image-linker-jobmanager:8081 -py /opt/app/job_linker.py && + echo 'Image-Linker job submitted successfully' && + sleep 1" + networks: + - ag_cloud + + flink-sounds-http-jobmanager: + build: + context: ./services/sounds_flink + dockerfile: Dockerfile + container_name: flink-sounds-http-jobmanager + command: jobmanager + ports: + - "8083:8081" + environment: + JOB_MANAGER_RPC_ADDRESS: flink-sounds-http-jobmanager + KAFKA_BROKERS: kafka:9092 + SOURCE_TOPIC: sound_new_sounds_connections + SINK_TOPIC: "" + GROUP_ID: flink-classifier-sounds + CLASSIFIER_HTTP_URL: http://sounds_classifier:8088/classify + DEFAULT_PARALLELISM: 2 + KAFKA_START: earliest + PYTHON: /opt/venv/bin/python + FLINK_PYTHON: /opt/venv/bin/python + networks: + - ag_cloud + + flink-sounds-http-taskmanager: + build: + context: ./services/sounds_flink + dockerfile: Dockerfile + container_name: flink-sounds-http-taskmanager + command: taskmanager + depends_on: + flink-sounds-http-jobmanager: + condition: service_started + environment: + JOB_MANAGER_RPC_ADDRESS: flink-sounds-http-jobmanager + PYTHON: /opt/venv/bin/python + FLINK_PYTHON: /opt/venv/bin/python + FLINK_PROPERTIES: |- + jobmanager.rpc.address: flink-sounds-http-jobmanager + taskmanager.numberOfTaskSlots: 2 + networks: + - ag_cloud + + flink-sounds-http-submit: + build: + context: ./services/sounds_flink + dockerfile: Dockerfile + container_name: flink-sounds-http-submit + depends_on: + flink-sounds-http-jobmanager: + condition: service_started + flink-sounds-http-taskmanager: + condition: service_started + command: + - /opt/flink/bin/flink + - run + - -d + - -m + - flink-sounds-http-jobmanager:8081 + - -Dpython.client.executable=/opt/venv/bin/python + - -Dpython.executable=/opt/venv/bin/python + - -py + - /opt/app/flink_job.py + environment: + JOB_MANAGER_RPC_ADDRESS: flink-sounds-http-jobmanager + KAFKA_BROKERS: kafka:9092 + SOURCE_TOPIC: sound_new_sounds_connections + SINK_TOPIC: "" + GROUP_ID: flink-classifier-sounds + CLASSIFIER_HTTP_URL: http://sounds_classifier:8088/classify + DEFAULT_PARALLELISM: 2 + KAFKA_START: earliest + PYTHON: /opt/venv/bin/python + FLINK_PYTHON: /opt/venv/bin/python + networks: + - ag_cloud From 9de8fefb92e5a08e1d3adda2830ec0fdf33d555b Mon Sep 17 00:00:00 2001 From: m0533199321 Date: Wed, 12 Nov 2025 19:50:42 +0200 Subject: [PATCH 2/2] change from an update compose --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 61799b947..f610d32ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1232,4 +1232,4 @@ services: PYTHON: /opt/venv/bin/python FLINK_PYTHON: /opt/venv/bin/python networks: - - ag_cloud + - ag_cloud \ No newline at end of file