diff --git a/docker-compose.yml b/docker-compose.yml index 9ebbac53a..ccef2cabd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ + # ========================== # Docker Compose - AG Cloud # ========================== @@ -10,8 +11,6 @@ networks: ag_cloud: name: ag_cloud driver: bridge - flink-net: - driver: bridge # -------------------------- # Volumes @@ -87,7 +86,7 @@ services: dockerfile: Dockerfile environment: - ADDR=0.0.0.0 - - PORT=8005 + - PORT=8003 - USE_UTC=false - WINDOW_MIN=1 - STABLE_SEC=1 @@ -101,7 +100,7 @@ services: command: [ "python", "-u", "src/metrics.py" ] ports: - - "8005:8005" + - "8003:8003" depends_on: - minio-hot networks: @@ -115,8 +114,10 @@ services: plant_stress_daily: build: ./services/plant_stress + env_file: + - ./services/plant_stress/.env + restart: "no" environment: - TZ: "Asia/Jerusalem" MODEL_DIR: /models CONFIDENCE_THRESHOLD: "0.60" TF_CPP_MIN_LOG_LEVEL: "2" @@ -128,17 +129,20 @@ services: MINIO_BUCKET: sound MINIO_PREFIX: plants/ MINIO_SECURE: "false" + DEFAULT_AREA: unknown DEFAULT_LAT: 0.0 DEFAULT_LON: 0.0 DEFAULT_IMAGE_URL: https://example.com/placeholder.jpg DEFAULT_VOD: https://example.com/placeholder.mp4 DEFAULT_HLS: https://example.com/placeholder.m3u8 + # ==== Alerts → Kafka ==== ENABLE_ALERTS: "true" KAFKA_BOOTSTRAP: "kafka:9092" ALERT_TOPIC: "alerts" ALERT_TYPE: "plant_drought_detected" KAFKA_CLIENT_ID: "plant-stress-producer" + command: ["python","-u","/app/predict_minio_daily.py"] volumes: - "./services/plant_stress/models:/models:ro" depends_on: @@ -151,7 +155,6 @@ services: kafka: condition: service_healthy networks: [ag_cloud] - restart: unless-stopped # ------------------------- # MQTT + Kafka + MQTT-router @@ -391,7 +394,7 @@ services: MINIO_NOTIFY_KAFKA_ENABLE_aerial: "on" MINIO_NOTIFY_KAFKA_BROKERS_aerial: "kafka:9092" MINIO_NOTIFY_KAFKA_TOPIC_aerial: "image.new.aerial" - + MINIO_NOTIFY_KAFKA_ENABLE_air: "on" MINIO_NOTIFY_KAFKA_BROKERS_air: "kafka:9092" MINIO_NOTIFY_KAFKA_TOPIC_air: "image.new.air" @@ -620,7 +623,7 @@ services: - mqtt_ingest networks: - ag_cloud - + mqtt_gateway: build: context: ./services/mqtt_gateway @@ -670,20 +673,18 @@ services: - HEAD=/app/classification/models/head/head_cnn14_rf.joblib - HEAD_META=/app/classification/models/head/head_cnn14_rf.joblib.meta.json - # DB + # DB - WRITE_DB=false - DB_URL=postgresql://missions_user:pg123@postgres:5432/missions_db - DB_SCHEMA=agcloud_audio - DB_RUN_ID=api-default - - FILES_SCHEMA=public - - FILES_TABLE=sound_new_sounds_connections - # Kafka + # Kafka - KAFKA_BROKERS=kafka:9092 - ALERTS_TOPIC=alerts - ENABLE_ALERTS=true - # MinIO + # MinIO - MINIO_ENDPOINT=minio-hot:9000 - MINIO_ACCESS_KEY=minioadmin - MINIO_SECRET_KEY=minioadmin123 @@ -694,7 +695,7 @@ services: - ALLOWED_CONTENT_TYPES=audio/wav,audio/x-wav,audio/mpeg,audio/flac,audio/ogg,audio/mp4 - MAX_BYTES=104857600 - # Tuning params + # Tuning params - UNKNOWN_THRESHOLD=0.4 - WINDOW_SEC=2.0 - HOP_SEC=0.5 @@ -708,11 +709,11 @@ services: mc-bootstrap: condition: service_started ports: - - "8088:8088" + - "8056:8056" networks: - ag_cloud healthcheck: - test: [ "CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8088/health').read()" ] + test: [ "CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8056/health').read()" ] interval: 45s timeout: 5s retries: 10 @@ -813,7 +814,7 @@ services: ports: - "8091:8088" restart: unless-stopped - + # -------------------------- # Flink JobManager & TaskManager # -------------------------- @@ -844,7 +845,7 @@ services: python.executable: /usr/bin/python3 - HTTP_INFER_URL=http://fruit-inference-http:8004/infer_json volumes: - - ./streaming/flink/jobs:/opt/flink/jobs:ro + - ./streaming/flink/jobs:/opt/flink/jobs - ./streaming/flink/connectors/flink-json-1.18.1.jar:/opt/flink/lib/flink-json-1.18.1.jar:ro - ./streaming/flink/connectors/flink-sql-connector-kafka-3.2.0-1.18.jar:/opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar:ro - ./streaming/flink/connectors/flink-connector-kafka-3.2.0-1.18.jar:/opt/flink/lib/flink-connector-kafka-3.2.0-1.18.jar:ro @@ -911,9 +912,9 @@ services: FLINK_PROPERTIES= jobmanager.rpc.address: flink-jobmanager parallelism.default: 2 - taskmanager.numberOfTaskSlots: 2 + taskmanager.numberOfTaskSlots: 4 jobmanager.memory.process.size: 1600m - taskmanager.memory.process.size: 1728m + taskmanager.memory.process.size: 2048m s3.endpoint: http://minio-hot:9000 s3.path.style.access: true s3.access.key: minioadmin @@ -1056,7 +1057,7 @@ services: networks: [ ag_cloud ] environment: - KAFKA_BOOTSTRAP=kafka:9092 - - INPUT_TOPIC=imagery.new.fruit + - INPUT_TOPIC=inference.dispatched.camera - TEAM=fruit - HTTP_URL=http://fruit-inference-http:8004/infer_json - DLQ_TOPIC=dlq.inference.http @@ -1071,9 +1072,9 @@ services: - ./streaming/flink/connectors/kafka-clients-3.2.3.jar:/opt/flink/lib/kafka-clients-3.2.3.jar:ro - ./streaming/flink/connectors/lz4-java-1.8.0.jar:/opt/flink/lib/lz4-java-1.8.0.jar:ro - ./streaming/flink/connectors/snappy-java-1.1.10.5.jar:/opt/flink/lib/snappy-java-1.1.10.5.jar:ro - command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic imagery.new.fruit --team fruit --http-url http://fruit-inference-http:8004/infer_json --group-id http-dispatcher-fruit --dlq-topic dlq.inference.http; tail -f /dev/null" ] + command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic inference.dispatched.camera --team fruit --http-url http://fruit-inference-http:8004/infer_json --group-id http-dispatcher-fruit --dlq-topic dlq.inference.http; tail -f /dev/null" ] restart: always - + flink-dispatcher-camera: image: agcloud-flink-py:1.18 container_name: flink-dispatcher-camera @@ -1084,7 +1085,7 @@ services: networks: [ag_cloud] environment: - KAFKA_BOOTSTRAP=kafka:9092 - - INPUT_TOPIC=imagery.new.camera + - INPUT_TOPIC=image.new.fruits - TEAM=camera - HTTP_URL=http://camera-inference-http:8004/infer_json - DLQ_TOPIC=dlq.inference.http @@ -1094,7 +1095,7 @@ services: volumes: - ./streaming/flink/jobs:/opt/flink/jobs:ro - ./streaming/flink/connectors:/opt/flink/lib/connectors:ro - command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/connectors/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic imagery.new.camera --team camera --http-url http://camera-inference-http:8004/infer_json --group-id http-dispatcher-camera --dlq-topic dlq.inference.http; tail -f /dev/null" ] + command: [ "bash", "-lc", "set -e; echo 'Waiting for JobManager to accept commands...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/connectors/flink-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-sql-connector-kafka-3.2.0-1.18.jar,file:///opt/flink/lib/connectors/flink-json-1.18.1.jar --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic image.new.fruits --team camera --http-url http://camera-inference-http:8004/infer_json --group-id http-dispatcher-camera --dlq-topic dlq.inference.http; tail -f /dev/null" ] restart: always flink-dispatcher-soil: @@ -1117,7 +1118,7 @@ services: - ./streaming/flink/jobs:/opt/flink/jobs:ro - ./streaming/flink/connectors:/opt/flink/lib/connectors:ro command: [ "bash", "-lc", "set -e; echo 'Waiting...'; until /opt/flink/bin/flink list --jobmanager flink-jobmanager:8081 >/dev/null 2>&1; do echo 'still waiting...'; sleep 3; done; echo 'JobManager is ready!'; /opt/flink/bin/flink run -Dpython.client.executable=/usr/bin/python3 -Dpython.executable=/usr/bin/python3 -Dpipeline.jars=file:///opt/flink/lib/connectors/... --jobmanager flink-jobmanager:8081 --detached --python /opt/flink/jobs/http_dispatcher.py -- --bootstrap kafka:9092 --input-topic image.new.ground --team soil_moisture --http-url http://soil-inference-http:8004/infer_json --group-id http-dispatcher-soil --dlq-topic dlq.inference.http; tail -f /dev/null" ] - + flink-alerts-job: build: @@ -1180,10 +1181,10 @@ services: container_name: alerts_gateway command: [ "uvicorn", "gateway:app", "--host", "0.0.0.0", "--port", "8000" ] ports: - - "8010:8000" + - "8010:8000" networks: - ag_cloud - + image-linker-jobmanager: build: context: ./services/image-linker @@ -1191,7 +1192,7 @@ services: container_name: image-linker-jobmanager command: jobmanager ports: - - "8084:8081" + - "8084:8081" environment: - JOB_MANAGER_RPC_ADDRESS=image-linker-jobmanager - KAFKA_BROKERS=kafka:9092 @@ -1238,7 +1239,7 @@ services: container_name: flink-sounds-http-jobmanager command: jobmanager ports: - - "8083:8081" + - "8083:8081" environment: JOB_MANAGER_RPC_ADDRESS: flink-sounds-http-jobmanager KAFKA_BROKERS: kafka:9092 @@ -1340,13 +1341,13 @@ services: sleep 5; done && echo '✅ Reports ready, submitting Flink job...' && - flink run -m localhost:8091 -py /opt/app/sensorAnomalyPro/app.py && + flink run -m localhost:8099 -py /opt/app/sensorAnomalyPro/app.py && tail -f /dev/null" depends_on: - sensor_anomaly_pro - kafka ports: - - "8091:8091" + - "8099:8099" environment: - JOB_MANAGER_RPC_ADDRESS=jobmanager - KAFKA_BROKERS=kafka:9092 @@ -1381,7 +1382,26 @@ services: - ag_cloud - + edge-sensors: + build: + context: ./mqtt_and_kafka/Sensor_edge_device + dockerfile: Dockerfile.edge + container_name: edge-sensors + depends_on: + mosquitto: + condition: service_healthy + environment: + - BROKER=mosquitto + - PORT=1883 + - TOPIC=dev-robot-telemetry-raw + networks: + - ag_cloud + restart: unless-stopped + + + + + vector_service: build: ./services/vector_service container_name: vector_service @@ -1409,14 +1429,14 @@ services: dockerfile: Dockerfile.flink container_name: sensorguard-jobmanager ports: - - "8081:8081" + - "8999:8999" command: > bash -c " /docker-entrypoint.sh jobmanager & echo 'Waiting for Flink JobManager startup...' && sleep 15 && echo 'Submitting sensorGuard Flink job...' && - flink run -m localhost:8081 -py /opt/app/main.py && + flink run -m localhost:8999 -py /opt/app/main.py && tail -f /dev/null" environment: - JOB_MANAGER_RPC_ADDRESS=sensorguard-jobmanager diff --git a/services/sensorGuard/sensorGuard/flink_app/api/devices_updater.py b/services/sensorGuard/sensorGuard/flink_app/api/devices_updater.py index 534d2893f..a6fd3ac3f 100644 --- a/services/sensorGuard/sensorGuard/flink_app/api/devices_updater.py +++ b/services/sensorGuard/sensorGuard/flink_app/api/devices_updater.py @@ -7,7 +7,7 @@ def update_device_last_seen(device_id: str): Updates the 'last_seen' field for a specific device in the devices_sensor table. Uses PATCH /api/tables/devices_sensor """ - api_base = "http://host.docker.internal:8001" + api_base = "http://db_api_service:8001" token = get_access_token(api_base) headers = { "X-Service-Token": token, diff --git a/services/sensorGuard/sensorGuard/flink_app/config/settings.py b/services/sensorGuard/sensorGuard/flink_app/config/settings.py index 59853c063..b822826c3 100644 --- a/services/sensorGuard/sensorGuard/flink_app/config/settings.py +++ b/services/sensorGuard/sensorGuard/flink_app/config/settings.py @@ -5,7 +5,7 @@ class Settings: # --- API Configuration --- - DEVICES_API_BASE = os.getenv("DEVICES_API_BASE", "http://host.docker.internal:8001") + DEVICES_API_BASE = os.getenv("DEVICES_API_BASE", "http://db_api_service:8001") DEVICES_API_TOKEN = os.getenv("DEVICES_API_TOKEN", None) # --- Kafka Configuration --- diff --git a/services/sensorGuard/sensorGuard/flink_app/core/engine.py b/services/sensorGuard/sensorGuard/flink_app/core/engine.py index c4669b16b..057ef75d1 100644 --- a/services/sensorGuard/sensorGuard/flink_app/core/engine.py +++ b/services/sensorGuard/sensorGuard/flink_app/core/engine.py @@ -22,7 +22,7 @@ def __init__(self, cfg, writer, state: StateStore | None = None): self.state = state or StateStore() # --- API info & persistent token --- - self.api_base = "http://host.docker.internal:8001" + self.api_base = "http://db_api_service:8001" self.token = get_access_token(self.api_base) if self.token: print("[ENGINE] Access token acquired successfully.") diff --git a/services/sensorGuard/sensorGuard/flink_app/main.py b/services/sensorGuard/sensorGuard/flink_app/main.py index b5292b58e..fe0c4b858 100644 --- a/services/sensorGuard/sensorGuard/flink_app/main.py +++ b/services/sensorGuard/sensorGuard/flink_app/main.py @@ -143,7 +143,7 @@ def main(): cfg_path = base_dir / "config" / "rules.yaml" # --- Load sensors from API --- - api_base = os.getenv("DEVICES_API_BASE", "http://host.docker.internal:8001") + api_base = os.getenv("DEVICES_API_BASE", "http://db_api_service:8001") print(f"[INIT] Fetching active sensors from {api_base}...") token = get_access_token(api_base)