From 9f23bf0419c33090f78d581d299aafc5c96e41b3 Mon Sep 17 00:00:00 2001 From: Oleg Lazari Date: Tue, 11 Nov 2025 01:42:40 -0500 Subject: [PATCH] added fixes for clearing db --- Scripts/clear-db.sh | 223 ++++++++++++++++++ .../Fuzzilli/Corpus/PostgreSQLCorpus.swift | 80 ++----- docker-compose.workers.yml | 72 ------ 3 files changed, 246 insertions(+), 129 deletions(-) create mode 100755 Scripts/clear-db.sh diff --git a/Scripts/clear-db.sh b/Scripts/clear-db.sh new file mode 100755 index 000000000..c4ee0d41d --- /dev/null +++ b/Scripts/clear-db.sh @@ -0,0 +1,223 @@ +#!/bin/bash + +# Clear PostgreSQL Database Script +# This script clears all data from the Fuzzilli PostgreSQL database +# while preserving the schema and lookup table seed data. + +# Don't exit on error - we want to continue even if some operations fail +set +e + +# Database connection parameters +DB_CONTAINER="fuzzilli-postgres-master" +DB_NAME="fuzzilli_master" +DB_USER="fuzzilli" +DB_PASSWORD="fuzzilli123" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +CYAN='\033[0;36m' +NC='\033[0m' # No Color + +# Function to check if Docker is available +check_docker() { + if ! command -v docker &> /dev/null; then + echo -e "${RED}Error: Docker command not found. Please install Docker.${NC}" + exit 1 + fi +} + +# Function to check if PostgreSQL container is running +check_container() { + if ! docker ps --format "table {{.Names}}" | grep -q "$DB_CONTAINER"; then + echo -e "${RED}Error: PostgreSQL container '$DB_CONTAINER' is not running${NC}" + echo "Available containers:" + docker ps --format "table {{.Names}}\t{{.Status}}" + exit 1 + fi +} + +# Function to run a query +run_query() { + local query="$1" + docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>&1 +} + +# Function to run a query silently (for operations that might fail) +run_query_silent() { + local query="$1" + docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>/dev/null || true +} + +# Function to get table row counts +get_table_counts() { + echo -e "${CYAN}Current database statistics:${NC}" + run_query " + SELECT + 'main' as table_name, COUNT(*) as row_count FROM main + UNION ALL + SELECT 'fuzzer', COUNT(*) FROM fuzzer + UNION ALL + SELECT 'program', COUNT(*) FROM program + UNION ALL + SELECT 'execution', COUNT(*) FROM execution + UNION ALL + SELECT 'coverage_detail', COUNT(*) FROM coverage_detail + UNION ALL + SELECT 'feedback_vector_detail', COUNT(*) FROM feedback_vector_detail + UNION ALL + SELECT 'crash_analysis', COUNT(*) FROM crash_analysis + UNION ALL + SELECT 'fuzzer_statistics', COALESCE((SELECT COUNT(*) FROM fuzzer_statistics), 0); + " | grep -v "row_count" | grep -v "^$" | grep -v "^-" | while read -r line; do + if [ -n "$line" ]; then + echo " $line" + fi + done +} + +# Main execution +main() { + echo -e "${CYAN}========================================${NC}" + echo -e "${CYAN} Fuzzilli Database Cleanup${NC}" + echo -e "${CYAN}========================================${NC}" + echo "" + + check_docker + check_container + + # Show current statistics + get_table_counts + echo "" + + # Find all fuzzer-X tables + fuzzer_tables=$(run_query "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'fuzzer-%' ORDER BY tablename;" | grep -v "tablename" | grep -v "^-" | grep -v "^$" | grep -v "^(" | awk 'NF>0 {print $1}' | tr '\n' ' ') + + # Safety confirmation + if [ "${1:-}" != "--yes" ] && [ "${1:-}" != "-y" ]; then + echo -e "${YELLOW}WARNING: This will delete ALL data from the database!${NC}" + echo -e "${YELLOW}The following will be cleared:${NC}" + echo " - All fuzzer instances (main table)" + echo " - All corpus programs (fuzzer table)" + echo " - All executed programs (program table)" + echo " - All execution records (execution table)" + echo " - All coverage details (coverage_detail table)" + echo " - All feedback vector details (feedback_vector_detail table)" + echo " - All crash analyses (crash_analysis table)" + echo " - All fuzzer statistics (fuzzer_statistics table)" + if [ -n "$fuzzer_tables" ]; then + echo -e "${YELLOW} - All fuzzer-X tables (will be DROPPED):${NC}" + for table in $fuzzer_tables; do + echo " - $table" + done + fi + echo "" + echo -e "${YELLOW}The following will be preserved:${NC}" + echo " - Database schema (tables, indexes, views)" + echo " - Lookup tables (execution_type, mutator_type, execution_outcome)" + echo "" + read -p "Are you sure you want to continue? (yes/no): " confirm + if [ "$confirm" != "yes" ]; then + echo -e "${GREEN}Operation cancelled.${NC}" + exit 0 + fi + fi + + echo -e "${YELLOW}Clearing database...${NC}" + echo "" + + # Clear data tables in order (respecting foreign key constraints) + # Start with child tables first, then parent tables + # Use CASCADE to handle foreign key dependencies + + echo -e "${CYAN}Clearing coverage_detail...${NC}" + run_query_silent "TRUNCATE TABLE coverage_detail CASCADE;" + + echo -e "${CYAN}Clearing feedback_vector_detail...${NC}" + run_query_silent "TRUNCATE TABLE feedback_vector_detail CASCADE;" + + echo -e "${CYAN}Clearing crash_analysis...${NC}" + run_query_silent "TRUNCATE TABLE crash_analysis CASCADE;" + + echo -e "${CYAN}Clearing execution...${NC}" + run_query_silent "TRUNCATE TABLE execution CASCADE;" + + echo -e "${CYAN}Clearing program...${NC}" + run_query_silent "TRUNCATE TABLE program CASCADE;" + + echo -e "${CYAN}Clearing fuzzer (corpus)...${NC}" + run_query_silent "TRUNCATE TABLE fuzzer CASCADE;" + + echo -e "${CYAN}Clearing fuzzer_statistics...${NC}" + run_query_silent "TRUNCATE TABLE fuzzer_statistics CASCADE;" + + echo -e "${CYAN}Clearing main (fuzzer instances)...${NC}" + run_query_silent "TRUNCATE TABLE main CASCADE;" + + # Drop all fuzzer-X tables if they exist + if [ -n "$fuzzer_tables" ]; then + echo -e "${CYAN}Dropping fuzzer-X tables...${NC}" + for table in $fuzzer_tables; do + if [ -n "$table" ]; then + echo " Dropping table: $table" + run_query_silent "DROP TABLE IF EXISTS \"$table\" CASCADE;" + fi + done + else + echo -e "${CYAN}No fuzzer-X tables found to drop.${NC}" + fi + + # Reset sequences + echo -e "${CYAN}Resetting sequences...${NC}" + run_query_silent "ALTER SEQUENCE IF EXISTS main_fuzzer_id_seq RESTART WITH 1;" + run_query_silent "ALTER SEQUENCE IF EXISTS execution_execution_id_seq RESTART WITH 1;" + run_query_silent "ALTER SEQUENCE IF EXISTS feedback_vector_detail_id_seq RESTART WITH 1;" + run_query_silent "ALTER SEQUENCE IF EXISTS coverage_detail_id_seq RESTART WITH 1;" + run_query_silent "ALTER SEQUENCE IF EXISTS crash_analysis_id_seq RESTART WITH 1;" + + echo "" + echo -e "${GREEN}Database cleared successfully!${NC}" + echo "" + + # Show final statistics + get_table_counts + echo "" + + echo -e "${CYAN}========================================${NC}" + echo -e "${CYAN} Cleanup Complete${NC}" + echo -e "${CYAN}========================================${NC}" +} + +# Handle command line arguments +case "${1:-}" in + "help"|"-h"|"--help") + echo "Usage: $0 [--yes|-y]" + echo "" + echo "Clears all data from the Fuzzilli PostgreSQL database." + echo "" + echo "Options:" + echo " --yes, -y Skip confirmation prompt" + echo " help, -h Show this help message" + echo "" + echo "This script will:" + echo " - Delete all fuzzer instances, programs, executions, and related data" + echo " - Drop all fuzzer-X tables (fuzzer-1, fuzzer-2, etc.)" + echo " - Preserve the database schema and lookup tables" + echo " - Reset all sequences to start from 1" + echo "" + echo "Note: Update DB_CONTAINER variable in script if your container has a different name" + ;; + "") + main + ;; + "--yes"|"-y") + main "$1" + ;; + *) + echo "Unknown option: $1" + echo "Use '$0 help' for usage information" + exit 1 + ;; +esac + diff --git a/Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift b/Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift index b639f9162..4ffc3e07a 100644 --- a/Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift +++ b/Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift @@ -33,8 +33,8 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { private var fuzzerRegistered = false private var fuzzerId: Int? // Master database fuzzer ID - /// Batch execution storage - private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose)] = [] + /// Batch execution storage - includes execution metadata + private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = [] private var executionBatchSize: Int // Dynamic batch size private let executionBatchLock = NSLock() @@ -178,7 +178,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { guard fuzzerId != nil else { return } // Commit pending executions - let queuedExecutions = executionBatchLock.withLock { + let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock { let queuedExecutions = pendingExecutions pendingExecutions.removeAll() return queuedExecutions @@ -390,10 +390,10 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { } /// Add execution to batch for later processing - private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose) { + private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose, executionData: ExecutionData) { // Use atomic operations to avoid blocking locks - let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]? = executionBatchLock.withLock { - pendingExecutions.append((program, aspects, executionType)) + let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]? = executionBatchLock.withLock { + pendingExecutions.append((program, aspects, executionType, executionData)) let currentBatchSize = executionBatchSize let shouldProcess = pendingExecutions.count >= currentBatchSize if shouldProcess { @@ -413,7 +413,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { } /// Process a batch of executions - private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]) async { + private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]) async { guard let fuzzerId = fuzzerId else { logger.error("Cannot process execution batch: fuzzer not registered") return @@ -428,7 +428,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { var uniquePrograms: [String: (Program, ExecutionMetadata)] = [:] var executionBatchData: [ExecutionBatchData] = [] - for (program, aspects, executionType) in batch { + for (program, aspects, executionType, execData) in batch { // Filter out test programs with FUZZILLI_CRASH (false positive crashes) if DatabaseUtils.containsFuzzilliCrash(program: program) { if enableLogging { @@ -464,7 +464,11 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { mutatorType: nil, outcome: aspects.outcome, coverage: coveragePct, - coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set() + executionTimeMs: Int(execData.execTime * 1000), + coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set(), + stdout: execData.stdout, + stderr: execData.stderr, + fuzzout: execData.fuzzout ) executionBatchData.append(executionData) } @@ -491,7 +495,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { /// Flush any pending executions in the batch private func flushExecutionBatch() { - let batch = executionBatchLock.withLock { + let batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock { let batch = pendingExecutions pendingExecutions.removeAll() return batch @@ -521,7 +525,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { } // Process any queued executions - let queuedExecutions = executionBatchLock.withLock { + let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock { let queuedExecutions = pendingExecutions pendingExecutions.removeAll() return queuedExecutions @@ -681,53 +685,15 @@ public class PostgreSQLCorpus: ComponentBase, Corpus { return } - do { - // Use the registered fuzzer ID - guard let fuzzerId = fuzzerId else { - if enableLogging { - self.logger.info("Cannot store execution: fuzzer not registered") - } - return - } - - // DEBUG: Log execution storage attempt - if enableLogging { - self.logger.info("Storing execution: fuzzerId=\(fuzzerId), outcome=\(executionData.outcome), execTime=\(executionData.execTime)") - } - - // Derive coverage percentage (0-100) from evaluator if available - let coveragePct: Double = { - if let coverageEvaluator = self.fuzzer.evaluator as? ProgramCoverageEvaluator { - return coverageEvaluator.currentScore * 100.0 - } else { - return 0.0 - } - }() - - // Store both program and execution in a single transaction to avoid foreign key issues - _ = try await storage.storeProgramAndExecution( - program: program, - fuzzerId: fuzzerId, - executionType: executionType, - outcome: executionData.outcome, - coverage: coveragePct, - executionTimeMs: Int(executionData.execTime * 1000), - stdout: executionData.stdout, - stderr: executionData.stderr, - fuzzout: executionData.fuzzout, - metadata: ExecutionMetadata(lastOutcome: DatabaseExecutionOutcome( - id: DatabaseUtils.mapExecutionOutcome(outcome: aspects.outcome), - outcome: aspects.outcome.description, - description: aspects.outcome.description - )) - ) - - if enableLogging { - self.logger.info("Successfully stored program and execution") + // Add to execution batch for batched storage (even if fuzzer not registered yet) + addToExecutionBatch(program, aspects, executionType: executionType, executionData: executionData) + + if enableLogging { + if fuzzerId == nil { + self.logger.info("Queued execution for later (fuzzer not registered yet)") + } else { + self.logger.info("Added execution to batch: outcome=\(executionData.outcome), execTime=\(executionData.execTime)") } - - } catch { - logger.error("Failed to store execution: \(String(reflecting: error))") } } diff --git a/docker-compose.workers.yml b/docker-compose.workers.yml index 5c9c1e56c..cc34bc872 100644 --- a/docker-compose.workers.yml +++ b/docker-compose.workers.yml @@ -177,81 +177,9 @@ services: memory: 1G - # Worker 6 - Fuzzilli Container - fuzzer-worker-6: - build: - context: /home/tropic/vrig/fuzzilli-vrig-proj/fuzzillai - dockerfile: Cloud/VRIG/Dockerfile.distributed - container_name: fuzzer-worker-6 - environment: - - POSTGRES_URL=postgresql://fuzzilli:fuzzilli123@postgres-master:5432/fuzzilli_master - - FUZZER_INSTANCE_NAME=fuzzer-6 - - TIMEOUT=2500 - - MIN_MUTATIONS_PER_SAMPLE=25 - - DEBUG_LOGGING=false - depends_on: - postgres-master: - condition: service_healthy - volumes: - - fuzzer_data_6:/home/app/Corpus - - /home/tropic/vrig/fuzzilli-vrig-proj/fuzzbuild:/home/app/fuzzbuild:ro - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "pgrep -f FuzzilliCli || exit 1"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - networks: - - fuzzing-network - deploy: - resources: - limits: - memory: 2G - reservations: - memory: 1G - - - # Worker 7 - Fuzzilli Container - fuzzer-worker-7: - build: - context: /home/tropic/vrig/fuzzilli-vrig-proj/fuzzillai - dockerfile: Cloud/VRIG/Dockerfile.distributed - container_name: fuzzer-worker-7 - environment: - - POSTGRES_URL=postgresql://fuzzilli:fuzzilli123@postgres-master:5432/fuzzilli_master - - FUZZER_INSTANCE_NAME=fuzzer-7 - - TIMEOUT=2500 - - MIN_MUTATIONS_PER_SAMPLE=25 - - DEBUG_LOGGING=false - depends_on: - postgres-master: - condition: service_healthy - volumes: - - fuzzer_data_7:/home/app/Corpus - - /home/tropic/vrig/fuzzilli-vrig-proj/fuzzbuild:/home/app/fuzzbuild:ro - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "pgrep -f FuzzilliCli || exit 1"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 60s - networks: - - fuzzing-network - deploy: - resources: - limits: - memory: 2G - reservations: - memory: 1G - - volumes: fuzzer_data_1: fuzzer_data_2: fuzzer_data_3: fuzzer_data_4: fuzzer_data_5: - fuzzer_data_6: - fuzzer_data_7: