From e63bc79fd12be65ae74102e8d4d210093ca82b65 Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:10:08 +0900 Subject: [PATCH 1/9] Update docker-compose.yml --- docker-compose.yml | 85 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12fcaa6..529389e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,44 @@ -version: "3.8" - services: - sflow_collector: - image: pmacct/sfacctd:latest - restart: always - network_mode: host + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" volumes: - - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro - - ./networks.lst:/etc/pmacct/networks.lst:ro - - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro - - ./pretag.map:/etc/pmacct/pretag.map:ro + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + networks: + - monitoring_net - netflow_collector: - image: pmacct/nfacctd:latest + sflow-collector: + image: pmacct/sfacctd:latest + container_name: sflow-collector + depends_on: + - kafka restart: always - network_mode: host + ports: + - "6343:6343/udp" volumes: - - ./nfacctd.conf:/etc/pmacct/nfacctd.conf:ro + - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro - ./networks.lst:/etc/pmacct/networks.lst:ro - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro - ./pretag.map:/etc/pmacct/pretag.map:ro + networks: + - monitoring_net flow-consumer: image: serverforge/flow-consumer:v0.0.8-beta + container_name: flow-consumer depends_on: - - kafka - - clickhouse - network_mode: host + - kafka + - zookeeper + network_mode: "service:clickhouse" restart: always environment: - - brokers=localhost:9092 + - brokers=kafka:19092 - topic=pmacct.acct - database=dankflows - policy=365 DAYS @@ -38,30 +47,54 @@ services: image: zookeeper container_name: zookeeper restart: always - network_mode: host environment: - ALLOW_ANONYMOUS_LOGIN=yes + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] + interval: 5s + timeout: 10s + retries: 3 + networks: + - monitoring_net kafka: - image: ubuntu/kafka + image: wurstmeister/kafka container_name: kafka + hostname: kafka restart: always depends_on: - - zookeeper - network_mode: host + zookeeper: + condition: service_healthy environment: - - KAFKA_CREATE_TOPICS="pmacct.acct:3:1" - - KAFKA_ZOOKEEPER_CONNECT=127.0.0.1:2181 + - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 + - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE + - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_LOG_RETENTION_HOURS=1 - - ALLOW_PLAINTEXT_LISTENER=yes + networks: + - monitoring_net clickhouse: image: clickhouse/clickhouse-server container_name: clickhouse restart: always - network_mode: host volumes: - ./clickhouse-data:/var/lib/clickhouse/ - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml + networks: + - monitoring_net + +volumes: + grafana_data: + +networks: + monitoring_net: + driver: bridge From eb7e05f7874d454dd27e79b62016d4b094cc8160 Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:10:42 +0900 Subject: [PATCH 2/9] Update sfacctd.conf --- sfacctd.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sfacctd.conf b/sfacctd.conf index 6459ac9..6c67e58 100644 --- a/sfacctd.conf +++ b/sfacctd.conf @@ -1,7 +1,9 @@ plugins: kafka aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label kafka_output: json -kafka_broker_host: 127.0.0.1 +kafka_broker_host: kafka +kafka_broker_port: 19092 + kafka_topic: pmacct.acct kafka_refresh_time: 5 kafka_history: 5m From 2c11a6fd62af28527b7dbbd3bbba2eabdb4e7a7d Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:11:27 +0900 Subject: [PATCH 3/9] Create .env --- .env | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..7570dd1 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +GRAFANA_USERNAME="" +GRAFANA_PASSWORD="" From 8882e4b2341e24b2e46e9d1b07ac799d6c3355de Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:18:04 +0900 Subject: [PATCH 4/9] Create docker-compose-sflow.yaml --- docker-compose-sflow.yaml | 100 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docker-compose-sflow.yaml diff --git a/docker-compose-sflow.yaml b/docker-compose-sflow.yaml new file mode 100644 index 0000000..529389e --- /dev/null +++ b/docker-compose-sflow.yaml @@ -0,0 +1,100 @@ +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + networks: + - monitoring_net + + sflow-collector: + image: pmacct/sfacctd:latest + container_name: sflow-collector + depends_on: + - kafka + restart: always + ports: + - "6343:6343/udp" + volumes: + - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro + - ./networks.lst:/etc/pmacct/networks.lst:ro + - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro + - ./pretag.map:/etc/pmacct/pretag.map:ro + networks: + - monitoring_net + + flow-consumer: + image: serverforge/flow-consumer:v0.0.8-beta + container_name: flow-consumer + depends_on: + - kafka + - zookeeper + network_mode: "service:clickhouse" + restart: always + environment: + - brokers=kafka:19092 + - topic=pmacct.acct + - database=dankflows + - policy=365 DAYS + + zookeeper: + image: zookeeper + container_name: zookeeper + restart: always + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] + interval: 5s + timeout: 10s + retries: 3 + networks: + - monitoring_net + + kafka: + image: wurstmeister/kafka + container_name: kafka + hostname: kafka + restart: always + depends_on: + zookeeper: + condition: service_healthy + environment: + - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 + - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE + - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LOG_RETENTION_HOURS=1 + networks: + - monitoring_net + + clickhouse: + image: clickhouse/clickhouse-server + container_name: clickhouse + restart: always + volumes: + - ./clickhouse-data:/var/lib/clickhouse/ + - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml + - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml + - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml + networks: + - monitoring_net + +volumes: + grafana_data: + +networks: + monitoring_net: + driver: bridge From c76bb8f9af4519066cd1887ee2bd13c8e08202df Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:20:20 +0900 Subject: [PATCH 5/9] Update docker-compose-sflow.yaml --- docker-compose-sflow.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose-sflow.yaml b/docker-compose-sflow.yaml index 529389e..6a219e4 100644 --- a/docker-compose-sflow.yaml +++ b/docker-compose-sflow.yaml @@ -20,6 +20,7 @@ services: - kafka restart: always ports: + - "179:179" - "6343:6343/udp" volumes: - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro From 3fe563adabe505ebbf0acf58239b5ad2dd72f3cb Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:23:08 +0900 Subject: [PATCH 6/9] Create docker-compose-netflow.yaml --- docker-compose-netflow.yaml | 101 ++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 docker-compose-netflow.yaml diff --git a/docker-compose-netflow.yaml b/docker-compose-netflow.yaml new file mode 100644 index 0000000..8da83bf --- /dev/null +++ b/docker-compose-netflow.yaml @@ -0,0 +1,101 @@ +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + networks: + - monitoring_net + + netflow-collector: + image: pmacct/nfacctd:latest + container_name: netflow-collector + depends_on: + - kafka + restart: always + ports: + - "179:179" + - "2055:2055/udp" + volumes: + - ./nfacctd.conf:/etc/pmacct/nfacctd.conf:ro + - ./networks.lst:/etc/pmacct/networks.lst:ro + - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro + - ./pretag.map:/etc/pmacct/pretag.map:ro + networks: + - monitoring_net + + flow-consumer: + image: serverforge/flow-consumer:v0.0.8-beta + container_name: flow-consumer + depends_on: + - kafka + - zookeeper + network_mode: "service:clickhouse" + restart: always + environment: + - brokers=kafka:19092 + - topic=pmacct.acct + - database=dankflows + - policy=365 DAYS + + zookeeper: + image: zookeeper + container_name: zookeeper + restart: always + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] + interval: 5s + timeout: 10s + retries: 3 + networks: + - monitoring_net + + kafka: + image: wurstmeister/kafka + container_name: kafka + hostname: kafka + restart: always + depends_on: + zookeeper: + condition: service_healthy + environment: + - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 + - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE + - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LOG_RETENTION_HOURS=1 + networks: + - monitoring_net + + clickhouse: + image: clickhouse/clickhouse-server + container_name: clickhouse + restart: always + volumes: + - ./clickhouse-data:/var/lib/clickhouse/ + - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml + - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml + - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml + networks: + - monitoring_net + +volumes: + grafana_data: + +networks: + monitoring_net: + driver: bridge From 5d10028d949ab25e8b02e9aabb4e7a59c5393122 Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:25:24 +0900 Subject: [PATCH 7/9] Update nfacctd.conf --- nfacctd.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nfacctd.conf b/nfacctd.conf index 8e85cff..587cfa6 100644 --- a/nfacctd.conf +++ b/nfacctd.conf @@ -1,7 +1,9 @@ plugins: kafka aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label kafka_output: json -kafka_broker_host: 127.0.0.1 +kafka_broker_host: kafka +kafka_broker_port: 19092 + kafka_topic: pmacct.acct kafka_refresh_time: 5 kafka_history: 5m From fc565cd53829bac859ecf30de1f64e28133497fd Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:33:42 +0900 Subject: [PATCH 8/9] Create set-clickhouse-user.sh --- set-clickhouse-user.sh | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 set-clickhouse-user.sh diff --git a/set-clickhouse-user.sh b/set-clickhouse-user.sh new file mode 100644 index 0000000..3421c96 --- /dev/null +++ b/set-clickhouse-user.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# --- Configuration --- +# Default database to assign to the user. +DEFAULT_DATABASE="dankflows" + +# --- Script Logic --- +echo "--- ClickHouse User Setup Script ---" + +# Prompt for ClickHouse Username +read -p "Enter the desired ClickHouse username: " CLICKHOUSE_USERNAME +if [ -z "$CLICKHOUSE_USERNAME" ]; then + echo "Error: Username cannot be empty. Exiting." + exit 1 +fi + +# Prompt for ClickHouse Password +echo "Enter the password for the new ClickHouse user (40+ characters recommended):" +read -s -p "Password: " CLICKHOUSE_PASSWORD +echo # Add a newline after the silent password input + +if [ -z "$CLICKHOUSE_PASSWORD" ]; then + echo "Error: Password cannot be empty. Exiting." + exit 1 +fi + +echo "" # Add a newline for better readability +echo "Username to be created: $CLICKHOUSE_USERNAME" + +# 1. Generate SHA256 hash of the password +echo "Generating SHA256 hash for the password..." +PASSWORD_HASH=$(echo -n "$CLICKHOUSE_PASSWORD" | sha256sum | awk '{print $1}') + +if [ -z "$PASSWORD_HASH" ]; then + echo "Error: Failed to generate password hash. Exiting." + exit 1 +fi + +echo "Password SHA256 Hash: $PASSWORD_HASH" + +# 2. Connect to ClickHouse and create user +echo "Connecting to ClickHouse to create user '$CLICKHOUSE_USERNAME'..." + +# Create the SQL commands +SQL_COMMANDS="CREATE USER IF NOT EXISTS ${CLICKHOUSE_USERNAME} IDENTIFIED WITH SHA256_HASH BY '${PASSWORD_HASH}' DEFAULT DATABASE ${DEFAULT_DATABASE}; +GRANT ALL ON ${DEFAULT_DATABASE}.* TO ${CLICKHOUSE_USERNAME};" + +# Execute the SQL commands +echo "$SQL_COMMANDS" | docker compose exec -T clickhouse clickhouse-client --multiquery + +if [ $? -eq 0 ]; then + echo "User '$CLICKHOUSE_USERNAME' created and granted permissions successfully." +else + echo "Error: Failed to create or configure ClickHouse user. Please check the logs above." + exit 1 +fi + +echo "--- Setup Complete ---" +echo "Remember to store your chosen password securely." From 25db198d9b8ce422016111cdbc0a10d8581abdba Mon Sep 17 00:00:00 2001 From: SeongUk Moon Date: Sun, 25 May 2025 16:37:16 +0900 Subject: [PATCH 9/9] Revert docker-compose.yml --- docker-compose.yml | 85 ++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 59 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 529389e..12fcaa6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,44 +1,35 @@ -services: - grafana: - image: grafana/grafana:latest - container_name: grafana - restart: unless-stopped - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - environment: - - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} - networks: - - monitoring_net +version: "3.8" - sflow-collector: +services: + sflow_collector: image: pmacct/sfacctd:latest - container_name: sflow-collector - depends_on: - - kafka restart: always - ports: - - "6343:6343/udp" + network_mode: host volumes: - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro - ./networks.lst:/etc/pmacct/networks.lst:ro - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro - ./pretag.map:/etc/pmacct/pretag.map:ro - networks: - - monitoring_net + + netflow_collector: + image: pmacct/nfacctd:latest + restart: always + network_mode: host + volumes: + - ./nfacctd.conf:/etc/pmacct/nfacctd.conf:ro + - ./networks.lst:/etc/pmacct/networks.lst:ro + - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro + - ./pretag.map:/etc/pmacct/pretag.map:ro flow-consumer: image: serverforge/flow-consumer:v0.0.8-beta - container_name: flow-consumer depends_on: - - kafka - - zookeeper - network_mode: "service:clickhouse" + - kafka + - clickhouse + network_mode: host restart: always environment: - - brokers=kafka:19092 + - brokers=localhost:9092 - topic=pmacct.acct - database=dankflows - policy=365 DAYS @@ -47,54 +38,30 @@ services: image: zookeeper container_name: zookeeper restart: always + network_mode: host environment: - ALLOW_ANONYMOUS_LOGIN=yes - ulimits: - nofile: - soft: 65536 - hard: 65536 - healthcheck: - test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] - interval: 5s - timeout: 10s - retries: 3 - networks: - - monitoring_net kafka: - image: wurstmeister/kafka + image: ubuntu/kafka container_name: kafka - hostname: kafka restart: always depends_on: - zookeeper: - condition: service_healthy + - zookeeper + network_mode: host environment: - - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 - - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT - - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE - - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_CREATE_TOPICS="pmacct.acct:3:1" + - KAFKA_ZOOKEEPER_CONNECT=127.0.0.1:2181 - KAFKA_LOG_RETENTION_HOURS=1 - networks: - - monitoring_net + - ALLOW_PLAINTEXT_LISTENER=yes clickhouse: image: clickhouse/clickhouse-server container_name: clickhouse restart: always + network_mode: host volumes: - ./clickhouse-data:/var/lib/clickhouse/ - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml - networks: - - monitoring_net - -volumes: - grafana_data: - -networks: - monitoring_net: - driver: bridge