diff --git a/.env b/.env new file mode 100644 index 0000000..7570dd1 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +GRAFANA_USERNAME="" +GRAFANA_PASSWORD="" diff --git a/docker-compose-netflow.yaml b/docker-compose-netflow.yaml new file mode 100644 index 0000000..8da83bf --- /dev/null +++ b/docker-compose-netflow.yaml @@ -0,0 +1,101 @@ +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + networks: + - monitoring_net + + netflow-collector: + image: pmacct/nfacctd:latest + container_name: netflow-collector + depends_on: + - kafka + restart: always + ports: + - "179:179" + - "2055:2055/udp" + volumes: + - ./nfacctd.conf:/etc/pmacct/nfacctd.conf:ro + - ./networks.lst:/etc/pmacct/networks.lst:ro + - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro + - ./pretag.map:/etc/pmacct/pretag.map:ro + networks: + - monitoring_net + + flow-consumer: + image: serverforge/flow-consumer:v0.0.8-beta + container_name: flow-consumer + depends_on: + - kafka + - zookeeper + network_mode: "service:clickhouse" + restart: always + environment: + - brokers=kafka:19092 + - topic=pmacct.acct + - database=dankflows + - policy=365 DAYS + + zookeeper: + image: zookeeper + container_name: zookeeper + restart: always + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] + interval: 5s + timeout: 10s + retries: 3 + networks: + - monitoring_net + + kafka: + image: wurstmeister/kafka + container_name: kafka + hostname: kafka + restart: always + depends_on: + zookeeper: + condition: service_healthy + environment: + - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 + - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE + - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LOG_RETENTION_HOURS=1 + networks: + - monitoring_net + + clickhouse: + image: clickhouse/clickhouse-server + container_name: clickhouse + restart: always + volumes: + - ./clickhouse-data:/var/lib/clickhouse/ + - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml + - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml + - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml + networks: + - monitoring_net + +volumes: + grafana_data: + +networks: + monitoring_net: + driver: bridge diff --git a/docker-compose-sflow.yaml b/docker-compose-sflow.yaml new file mode 100644 index 0000000..6a219e4 --- /dev/null +++ b/docker-compose-sflow.yaml @@ -0,0 +1,101 @@ +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + ports: + - "3000:3000" + volumes: + - grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME} + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD} + networks: + - monitoring_net + + sflow-collector: + image: pmacct/sfacctd:latest + container_name: sflow-collector + depends_on: + - kafka + restart: always + ports: + - "179:179" + - "6343:6343/udp" + volumes: + - ./sfacctd.conf:/etc/pmacct/sfacctd.conf:ro + - ./networks.lst:/etc/pmacct/networks.lst:ro + - ./peering_agent.map:/etc/pmacct/peering_agent.map:ro + - ./pretag.map:/etc/pmacct/pretag.map:ro + networks: + - monitoring_net + + flow-consumer: + image: serverforge/flow-consumer:v0.0.8-beta + container_name: flow-consumer + depends_on: + - kafka + - zookeeper + network_mode: "service:clickhouse" + restart: always + environment: + - brokers=kafka:19092 + - topic=pmacct.acct + - database=dankflows + - policy=365 DAYS + + zookeeper: + image: zookeeper + container_name: zookeeper + restart: always + environment: + - ALLOW_ANONYMOUS_LOGIN=yes + ulimits: + nofile: + soft: 65536 + hard: 65536 + healthcheck: + test: ["CMD-SHELL", "echo ruok | nc -w 2 zookeeper 2181"] + interval: 5s + timeout: 10s + retries: 3 + networks: + - monitoring_net + + kafka: + image: wurstmeister/kafka + container_name: kafka + hostname: kafka + restart: always + depends_on: + zookeeper: + condition: service_healthy + environment: + - KAFKA_ADVERTISED_LISTENERS=INSIDE://:19092,OUTSIDE://172.17.0.1:9092 + - KAFKA_LISTENERS=INSIDE://kafka:19092,OUTSIDE://:9092 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT + - KAFKA_INTER_BROKER_LISTENER_NAME=INSIDE + - KAFKA_CREATE_TOPICS=pmacct.acct:3:1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LOG_RETENTION_HOURS=1 + networks: + - monitoring_net + + clickhouse: + image: clickhouse/clickhouse-server + container_name: clickhouse + restart: always + volumes: + - ./clickhouse-data:/var/lib/clickhouse/ + - ./no_logging.xml:/etc/clickhouse-server/users.d/no_logging.xml + - ./disable_all_the_logs.xml:/etc/clickhouse-server/config.d/disable_all_the_logs.xml + - ./no_verbose.xml:/etc/clickhouse-server/config.d/no_verbose.xml + networks: + - monitoring_net + +volumes: + grafana_data: + +networks: + monitoring_net: + driver: bridge diff --git a/nfacctd.conf b/nfacctd.conf index 8e85cff..587cfa6 100644 --- a/nfacctd.conf +++ b/nfacctd.conf @@ -1,7 +1,9 @@ plugins: kafka aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label kafka_output: json -kafka_broker_host: 127.0.0.1 +kafka_broker_host: kafka +kafka_broker_port: 19092 + kafka_topic: pmacct.acct kafka_refresh_time: 5 kafka_history: 5m diff --git a/set-clickhouse-user.sh b/set-clickhouse-user.sh new file mode 100644 index 0000000..3421c96 --- /dev/null +++ b/set-clickhouse-user.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# --- Configuration --- +# Default database to assign to the user. +DEFAULT_DATABASE="dankflows" + +# --- Script Logic --- +echo "--- ClickHouse User Setup Script ---" + +# Prompt for ClickHouse Username +read -p "Enter the desired ClickHouse username: " CLICKHOUSE_USERNAME +if [ -z "$CLICKHOUSE_USERNAME" ]; then + echo "Error: Username cannot be empty. Exiting." + exit 1 +fi + +# Prompt for ClickHouse Password +echo "Enter the password for the new ClickHouse user (40+ characters recommended):" +read -s -p "Password: " CLICKHOUSE_PASSWORD +echo # Add a newline after the silent password input + +if [ -z "$CLICKHOUSE_PASSWORD" ]; then + echo "Error: Password cannot be empty. Exiting." + exit 1 +fi + +echo "" # Add a newline for better readability +echo "Username to be created: $CLICKHOUSE_USERNAME" + +# 1. Generate SHA256 hash of the password +echo "Generating SHA256 hash for the password..." +PASSWORD_HASH=$(echo -n "$CLICKHOUSE_PASSWORD" | sha256sum | awk '{print $1}') + +if [ -z "$PASSWORD_HASH" ]; then + echo "Error: Failed to generate password hash. Exiting." + exit 1 +fi + +echo "Password SHA256 Hash: $PASSWORD_HASH" + +# 2. Connect to ClickHouse and create user +echo "Connecting to ClickHouse to create user '$CLICKHOUSE_USERNAME'..." + +# Create the SQL commands +SQL_COMMANDS="CREATE USER IF NOT EXISTS ${CLICKHOUSE_USERNAME} IDENTIFIED WITH SHA256_HASH BY '${PASSWORD_HASH}' DEFAULT DATABASE ${DEFAULT_DATABASE}; +GRANT ALL ON ${DEFAULT_DATABASE}.* TO ${CLICKHOUSE_USERNAME};" + +# Execute the SQL commands +echo "$SQL_COMMANDS" | docker compose exec -T clickhouse clickhouse-client --multiquery + +if [ $? -eq 0 ]; then + echo "User '$CLICKHOUSE_USERNAME' created and granted permissions successfully." +else + echo "Error: Failed to create or configure ClickHouse user. Please check the logs above." + exit 1 +fi + +echo "--- Setup Complete ---" +echo "Remember to store your chosen password securely." diff --git a/sfacctd.conf b/sfacctd.conf index 6459ac9..6c67e58 100644 --- a/sfacctd.conf +++ b/sfacctd.conf @@ -1,7 +1,9 @@ plugins: kafka aggregate: src_host, dst_host, src_port, dst_port, src_as, dst_as, proto, tag, label kafka_output: json -kafka_broker_host: 127.0.0.1 +kafka_broker_host: kafka +kafka_broker_port: 19092 + kafka_topic: pmacct.acct kafka_refresh_time: 5 kafka_history: 5m