docker-compose-streaming.yml

version: '3.8'

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.5.0
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - datalakehouse

  kafka:
    image: confluentinc/cp-kafka:7.5.0
    hostname: kafka
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://kafka:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
    volumes:
      - ./example_data:/kafka-example-data
    networks:
      - datalakehouse

  kafka-connect:
    image: confluentinc/cp-kafka-connect:7.5.0
    depends_on:
      - kafka
      - minio
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka:29092'
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: "connect-cluster"
      CONNECT_CONFIG_STORAGE_TOPIC: "connect-configs"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_STORAGE_TOPIC: "connect-offsets"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: "connect-status"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.StringConverter"
      CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components/,/connectors/'
    command: 
      - bash 
      - -c 
      - |
        #
        echo "Installing connector plugins"
        confluent-hub install --no-prompt tabular/iceberg-kafka-connect:0.6.19
        #
        echo "Launching Kafka Connect worker"
        /etc/confluent/docker/run & 
        #
        echo "Waiting for Kafka Connect to start listening on localhost ⏳"
        while : ; do
          curl_status=$$(curl -s -o /dev/null -w %{http_code} http://localhost:8083/connectors)
          echo -e $$(date) " Kafka Connect listener HTTP state: " $$curl_status " (waiting for 200)"
          if [ $$curl_status -eq 200 ] ; then
            break
          fi
          sleep 5 
        done
        echo -e "\n--\n+> Creating connector"
        curl -X PUT \
        -H 'Content-Type: application/json' \
        -H 'Accept: application/json' http://localhost:8083/connectors/iceberg-sink-connector/config \
        -d '{
            "connector.class": "io.tabular.iceberg.connect.IcebergSinkConnector",
            "iceberg.catalog.s3.endpoint":"http://minio:9000",
            "iceberg.catalog.s3.secret-access-key":"minioadmin",
            "iceberg.catalog.s3.access-key-id":"minioadmin",
            "iceberg.catalog.s3.path-style-access": "true",
            "iceberg.catalog.catalog-impl": "org.apache.iceberg.nessie.NessieCatalog",
            "iceberg.catalog.io-impl":"org.apache.iceberg.aws.s3.S3FileIO",
            "iceberg.control.commit.interval-ms":"1000",
            "iceberg.catalog.client.region":"us-south-1",
            "iceberg.catalog.uri":"http://nessie:19120/api/v1",
            "iceberg.tables.auto-create-enabled":"true",
            "iceberg.catalog.ref":"main",
            "tasks.max": "1",
            "topics": "user",
            "iceberg.tables": "user_kafka",
            "iceberg.catalog.warehouse": "s3a://nessie",
            "key.converter": "org.apache.kafka.connect.storage.StringConverter",
            "value.converter": "org.apache.kafka.connect.json.JsonConverter",
            "value.converter.schemas.enable": "false",
            "schemas.enable": "false"

        }'
        sleep infinity
    volumes:
      - ./kafka-connect-plugins:/etc/kafka-connect/jars
    networks:
      datalakehouse:

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafka-ui
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: "local-cluster"
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: "kafka:9092" # il nome del servizio Kafka all'interno del network
      KAFKA_CLUSTERS_0_ZOOKEEPER: "zookeeper:2181"
    depends_on:
      - kafka
      - zookeeper
    networks:
      - datalakehouse


  minio:
    image: minio/minio:latest
    container_name: minio
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=minioadmin
    command: server /data --console-address ":9001"
    networks:
      datalakehouse:
    volumes:
      - ./minio_data:/data
      
  dremio:
    image: dremio/dremio-oss:latest
    container_name: dremio
    ports:
      - "9047:9047"    # Web UI
      - "31010:31010"  # ODBC
      - "32010:32010"  # JDBC
      - "45678:45678"  # Internal coordination
    environment:
      - DREMIO_JAVA_SERVER_EXTRA_OPTS=-Dpaths.dist=file:///opt/dremio/data/dist
    volumes:
      - ./dremio:/var/lib/dremio
      - ./dremio_data:/opt/dremio/data
    networks:
      datalakehouse:
    depends_on:
      - minio
      - nessie
  
  createbuckets:
    image: minio/mc
    depends_on:
      - minio
    entrypoint: >
      /bin/sh -c "
      /usr/bin/mc config host add minio http://minio:9000 minioadmin minioadmin;
      /usr/bin/mc rm -r --force minio/nessie;
      /usr/bin/mc mb minio/nessie;
      exit 0;
      "
    networks:
      datalakehouse:

  nessie:
    image: projectnessie/nessie:latest
    container_name: nessie
    ports:
      - "19120:19120"
    environment:
      - QUARKUS_PROFILE=prod
      - QUARKUS_HTTP_PORT=19120
    networks:
      datalakehouse:

volumes:
  dremio:
  dremio_data:
  minio_data:
  kafka_data:
  zookeeper_data:

networks:
  datalakehouse: