Added Grafana and improved docker-compose.yaml

.env

@@ -0,0 +1,2 @@
+GRAFANA_USERNAME=""
+GRAFANA_PASSWORD=""

docker-compose-netflow.yaml

@@ -0,0 +1,101 @@
+services:
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
+    networks:
+      - monitoring_net
+
+  netflow-collector:
+    image: pmacct/nfacctd:latest
+    container_name: netflow-collector
+    depends_on:
+      - kafka
+    restart: always
+    ports:
+      - "179:179"
+      - "2055:2055/udp"
+    volumes:
+      - ./nfacctd.conf:/etc/pmacct/nfacctd.conf:ro
+      - ./networks.lst:/etc/pmacct/networks.lst:ro
+      - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro
+      - ./pretag.map:/etc/pmacct/pretag.map:ro
+    networks:
+      - monitoring_net
+
+  flow-consumer:
+    image: serverforge/flow-consumer:v0.0.8-beta
+    container_name: flow-consumer
+    depends_on:
+      - kafka
+      - zookeeper
+    network_mode: "service:clickhouse"
+    restart: always
+    environment:
+      - brokers=kafka:19092
+      - topic=pmacct.acct
+      - database=dankflows
+      - policy=365 DAYS
+
+  zookeeper:
+    image: zookeeper
+    container_name: zookeeper
+    restart: always
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    healthcheck:
+      test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"]
+      interval: 5s
+      timeout: 10s
+      retries: 3
+    networks:
+      - monitoring_net
+
+  kafka:
+    image: wurstmeister/kafka
+    container_name: kafka
+    hostname: kafka
+    restart: always
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092
+      - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092
+      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
+      - KAFKA_CREATE_TOPICS=pmacct.acct:3:1
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LOG_RETENTION_HOURS=1
+    networks:
+      - monitoring_net
+
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    container_name: clickhouse
+    restart: always
+    volumes:
+      - ./clickhouse-data:/var/lib/clickhouse/
+      - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml
+      - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml
+      - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml
+    networks:
+      - monitoring_net
+
+volumes:
+  grafana_data:
+
+networks:
+  monitoring_net:
+    driver: bridge

docker-compose-sflow.yaml

@@ -0,0 +1,101 @@
+services:
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
+    networks:
+      - monitoring_net
+
+  sflow-collector:
+    image: pmacct/sfacctd:latest
+    container_name: sflow-collector
+    depends_on:
+      - kafka
+    restart: always
+    ports:
+      - "179:179"
+      - "6343:6343/udp"
+    volumes:
+      - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro
+      - ./networks.lst:/etc/pmacct/networks.lst:ro
+      - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro
+      - ./pretag.map:/etc/pmacct/pretag.map:ro
+    networks:
+      - monitoring_net
+
+  flow-consumer:
+    image: serverforge/flow-consumer:v0.0.8-beta
+    container_name: flow-consumer
+    depends_on:
+      - kafka
+      - zookeeper
+    network_mode: "service:clickhouse"
+    restart: always
+    environment:
+      - brokers=kafka:19092
+      - topic=pmacct.acct
+      - database=dankflows
+      - policy=365 DAYS
+
+  zookeeper:
+    image: zookeeper
+    container_name: zookeeper
+    restart: always
+    environment:
+      - ALLOW_ANONYMOUS_LOGIN=yes
+    ulimits:
+      nofile:
+        soft: 65536
+        hard: 65536
+    healthcheck:
+      test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"]
+      interval: 5s
+      timeout: 10s
+      retries: 3
+    networks:
+      - monitoring_net
+
+  kafka:
+    image: wurstmeister/kafka
+    container_name: kafka
+    hostname: kafka
+    restart: always
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+    environment:
+      - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092
+      - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092
+      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
+      - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE
+      - KAFKA_CREATE_TOPICS=pmacct.acct:3:1
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_LOG_RETENTION_HOURS=1
+    networks:
+      - monitoring_net
+
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    container_name: clickhouse
+    restart: always
+    volumes:
+      - ./clickhouse-data:/var/lib/clickhouse/
+      - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml
+      - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml
+      - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml
+    networks:
+      - monitoring_net
+
+volumes:
+  grafana_data:
+
+networks:
+  monitoring_net:
+    driver: bridge

nfacctd.conf

@@ -1,7 +1,9 @@
 plugins: kafka
 aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label
 kafka_output: json
-kafka_broker_host: 127.0.0.1
+kafka_broker_host: kafka
+kafka_broker_port: 19092
+
 kafka_topic: pmacct.acct
 kafka_refresh_time: 5
 kafka_history: 5m

set-clickhouse-user.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# --- Configuration ---
+# Default database to assign to the user.
+DEFAULT_DATABASE="dankflows"
+
+# --- Script Logic ---
+echo "--- ClickHouse User Setup Script ---"
+
+# Prompt for ClickHouse Username
+read -p "Enter the desired ClickHouse username: " CLICKHOUSE_USERNAME
+if [ -z "$CLICKHOUSE_USERNAME" ]; then
+    echo "Error: Username cannot be empty. Exiting."
+    exit 1
+fi
+
+# Prompt for ClickHouse Password
+echo "Enter the password for the new ClickHouse user (40+ characters recommended):"
+read -s -p "Password: " CLICKHOUSE_PASSWORD
+echo # Add a newline after the silent password input
+
+if [ -z "$CLICKHOUSE_PASSWORD" ]; then
+    echo "Error: Password cannot be empty. Exiting."
+    exit 1
+fi
+
+echo "" # Add a newline for better readability
+echo "Username to be created: $CLICKHOUSE_USERNAME"
+
+# 1. Generate SHA256 hash of the password
+echo "Generating SHA256 hash for the password..."
+PASSWORD_HASH=$(echo -n "$CLICKHOUSE_PASSWORD" | sha256sum | awk '{print $1}')
+
+if [ -z "$PASSWORD_HASH" ]; then
+    echo "Error: Failed to generate password hash. Exiting."
+    exit 1
+fi
+
+echo "Password SHA256 Hash: $PASSWORD_HASH"
+
+# 2. Connect to ClickHouse and create user
+echo "Connecting to ClickHouse to create user '$CLICKHOUSE_USERNAME'..."
+
+# Create the SQL commands
+SQL_COMMANDS="CREATE USER IF NOT EXISTS ${CLICKHOUSE_USERNAME} IDENTIFIED WITH SHA256_HASH BY '${PASSWORD_HASH}' DEFAULT DATABASE ${DEFAULT_DATABASE};
+GRANT ALL ON ${DEFAULT_DATABASE}.* TO ${CLICKHOUSE_USERNAME};"
+
+# Execute the SQL commands
+echo "$SQL_COMMANDS" | docker compose exec -T clickhouse clickhouse-client --multiquery
+
+if [ $? -eq 0 ]; then
+    echo "User '$CLICKHOUSE_USERNAME' created and granted permissions successfully."
+else
+    echo "Error: Failed to create or configure ClickHouse user. Please check the logs above."
+    exit 1
+fi
+
+echo "--- Setup Complete ---"
+echo "Remember to store your chosen password securely."

sfacctd.conf

@@ -1,7 +1,9 @@
 plugins: kafka
 aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label
 kafka_output: json
-kafka_broker_host: 127.0.0.1
+kafka_broker_host: kafka
+kafka_broker_port: 19092
+
 kafka_topic: pmacct.acct
 kafka_refresh_time: 5
 kafka_history: 5m