Skip to content

added fixes for clearing db #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
oleglazari merged 1 commit into from
Nov 11, 2025
Merged

added fixes for clearing db #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 223 additions & 0 deletions Scripts/clear-db.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,223 @@
#!/bin/bash

# Clear PostgreSQL Database Script
# This script clears all data from the Fuzzilli PostgreSQL database
# while preserving the schema and lookup table seed data.

# Don't exit on error - we want to continue even if some operations fail
set +e

# Database connection parameters
DB_CONTAINER="fuzzilli-postgres-master"
DB_NAME="fuzzilli_master"
DB_USER="fuzzilli"
DB_PASSWORD="fuzzilli123"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# Function to check if Docker is available
check_docker() {
if ! command -v docker &> /dev/null; then
echo -e "${RED}Error: Docker command not found. Please install Docker.${NC}"
exit 1
fi
}

# Function to check if PostgreSQL container is running
check_container() {
if ! docker ps --format "table {{.Names}}" | grep -q "$DB_CONTAINER"; then
echo -e "${RED}Error: PostgreSQL container '$DB_CONTAINER' is not running${NC}"
echo "Available containers:"
docker ps --format "table {{.Names}}\t{{.Status}}"
exit 1
fi
}

# Function to run a query
run_query() {
local query="$1"
docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>&1
}

# Function to run a query silently (for operations that might fail)
run_query_silent() {
local query="$1"
docker exec -i "$DB_CONTAINER" psql -U "$DB_USER" -d "$DB_NAME" -c "$query" 2>/dev/null || true
}

# Function to get table row counts
get_table_counts() {
echo -e "${CYAN}Current database statistics:${NC}"
run_query "
SELECT
'main' as table_name, COUNT(*) as row_count FROM main
UNION ALL
SELECT 'fuzzer', COUNT(*) FROM fuzzer
UNION ALL
SELECT 'program', COUNT(*) FROM program
UNION ALL
SELECT 'execution', COUNT(*) FROM execution
UNION ALL
SELECT 'coverage_detail', COUNT(*) FROM coverage_detail
UNION ALL
SELECT 'feedback_vector_detail', COUNT(*) FROM feedback_vector_detail
UNION ALL
SELECT 'crash_analysis', COUNT(*) FROM crash_analysis
UNION ALL
SELECT 'fuzzer_statistics', COALESCE((SELECT COUNT(*) FROM fuzzer_statistics), 0);
" | grep -v "row_count" | grep -v "^$" | grep -v "^-" | while read -r line; do
if [ -n "$line" ]; then
echo " $line"
fi
done
}

# Main execution
main() {
echo -e "${CYAN}========================================${NC}"
echo -e "${CYAN} Fuzzilli Database Cleanup${NC}"
echo -e "${CYAN}========================================${NC}"
echo ""

check_docker
check_container

# Show current statistics
get_table_counts
echo ""

# Find all fuzzer-X tables
fuzzer_tables=$(run_query "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename LIKE 'fuzzer-%' ORDER BY tablename;" | grep -v "tablename" | grep -v "^-" | grep -v "^$" | grep -v "^(" | awk 'NF>0 {print $1}' | tr '\n' ' ')

# Safety confirmation
if [ "${1:-}" != "--yes" ] && [ "${1:-}" != "-y" ]; then
echo -e "${YELLOW}WARNING: This will delete ALL data from the database!${NC}"
echo -e "${YELLOW}The following will be cleared:${NC}"
echo " - All fuzzer instances (main table)"
echo " - All corpus programs (fuzzer table)"
echo " - All executed programs (program table)"
echo " - All execution records (execution table)"
echo " - All coverage details (coverage_detail table)"
echo " - All feedback vector details (feedback_vector_detail table)"
echo " - All crash analyses (crash_analysis table)"
echo " - All fuzzer statistics (fuzzer_statistics table)"
if [ -n "$fuzzer_tables" ]; then
echo -e "${YELLOW} - All fuzzer-X tables (will be DROPPED):${NC}"
for table in $fuzzer_tables; do
echo " - $table"
done
fi
echo ""
echo -e "${YELLOW}The following will be preserved:${NC}"
echo " - Database schema (tables, indexes, views)"
echo " - Lookup tables (execution_type, mutator_type, execution_outcome)"
echo ""
read -p "Are you sure you want to continue? (yes/no): " confirm
if [ "$confirm" != "yes" ]; then
echo -e "${GREEN}Operation cancelled.${NC}"
exit 0
fi
fi

echo -e "${YELLOW}Clearing database...${NC}"
echo ""

# Clear data tables in order (respecting foreign key constraints)
# Start with child tables first, then parent tables
# Use CASCADE to handle foreign key dependencies

echo -e "${CYAN}Clearing coverage_detail...${NC}"
run_query_silent "TRUNCATE TABLE coverage_detail CASCADE;"

echo -e "${CYAN}Clearing feedback_vector_detail...${NC}"
run_query_silent "TRUNCATE TABLE feedback_vector_detail CASCADE;"

echo -e "${CYAN}Clearing crash_analysis...${NC}"
run_query_silent "TRUNCATE TABLE crash_analysis CASCADE;"

echo -e "${CYAN}Clearing execution...${NC}"
run_query_silent "TRUNCATE TABLE execution CASCADE;"

echo -e "${CYAN}Clearing program...${NC}"
run_query_silent "TRUNCATE TABLE program CASCADE;"

echo -e "${CYAN}Clearing fuzzer (corpus)...${NC}"
run_query_silent "TRUNCATE TABLE fuzzer CASCADE;"

echo -e "${CYAN}Clearing fuzzer_statistics...${NC}"
run_query_silent "TRUNCATE TABLE fuzzer_statistics CASCADE;"

echo -e "${CYAN}Clearing main (fuzzer instances)...${NC}"
run_query_silent "TRUNCATE TABLE main CASCADE;"

# Drop all fuzzer-X tables if they exist
if [ -n "$fuzzer_tables" ]; then
echo -e "${CYAN}Dropping fuzzer-X tables...${NC}"
for table in $fuzzer_tables; do
if [ -n "$table" ]; then
echo " Dropping table: $table"
run_query_silent "DROP TABLE IF EXISTS \"$table\" CASCADE;"
fi
done
else
echo -e "${CYAN}No fuzzer-X tables found to drop.${NC}"
fi

# Reset sequences
echo -e "${CYAN}Resetting sequences...${NC}"
run_query_silent "ALTER SEQUENCE IF EXISTS main_fuzzer_id_seq RESTART WITH 1;"
run_query_silent "ALTER SEQUENCE IF EXISTS execution_execution_id_seq RESTART WITH 1;"
run_query_silent "ALTER SEQUENCE IF EXISTS feedback_vector_detail_id_seq RESTART WITH 1;"
run_query_silent "ALTER SEQUENCE IF EXISTS coverage_detail_id_seq RESTART WITH 1;"
run_query_silent "ALTER SEQUENCE IF EXISTS crash_analysis_id_seq RESTART WITH 1;"

echo ""
echo -e "${GREEN}Database cleared successfully!${NC}"
echo ""

# Show final statistics
get_table_counts
echo ""

echo -e "${CYAN}========================================${NC}"
echo -e "${CYAN} Cleanup Complete${NC}"
echo -e "${CYAN}========================================${NC}"
}

# Handle command line arguments
case "${1:-}" in
"help"|"-h"|"--help")
echo "Usage: $0 [--yes|-y]"
echo ""
echo "Clears all data from the Fuzzilli PostgreSQL database."
echo ""
echo "Options:"
echo " --yes, -y Skip confirmation prompt"
echo " help, -h Show this help message"
echo ""
echo "This script will:"
echo " - Delete all fuzzer instances, programs, executions, and related data"
echo " - Drop all fuzzer-X tables (fuzzer-1, fuzzer-2, etc.)"
echo " - Preserve the database schema and lookup tables"
echo " - Reset all sequences to start from 1"
echo ""
echo "Note: Update DB_CONTAINER variable in script if your container has a different name"
;;
"")
main
;;
"--yes"|"-y")
main "$1"
;;
*)
echo "Unknown option: $1"
echo "Use '$0 help' for usage information"
exit 1
;;
esac

80 changes: 23 additions & 57 deletions Sources/Fuzzilli/Corpus/PostgreSQLCorpus.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
private var fuzzerRegistered = false
private var fuzzerId: Int? // Master database fuzzer ID

/// Batch execution storage
private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose)] = []
/// Batch execution storage - includes execution metadata
private var pendingExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = []
private var executionBatchSize: Int // Dynamic batch size
private let executionBatchLock = NSLock()

Expand Down Expand Up @@ -178,7 +178,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
guard fuzzerId != nil else { return }

// Commit pending executions
let queuedExecutions = executionBatchLock.withLock {
let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
let queuedExecutions = pendingExecutions
pendingExecutions.removeAll()
return queuedExecutions
Expand Down Expand Up @@ -390,10 +390,10 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
}

/// Add execution to batch for later processing
private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose) {
private func addToExecutionBatch(_ program: Program, _ aspects: ProgramAspects, executionType: DatabaseExecutionPurpose, executionData: ExecutionData) {
// Use atomic operations to avoid blocking locks
let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]? = executionBatchLock.withLock {
pendingExecutions.append((program, aspects, executionType))
let shouldProcessBatch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]? = executionBatchLock.withLock {
pendingExecutions.append((program, aspects, executionType, executionData))
let currentBatchSize = executionBatchSize
let shouldProcess = pendingExecutions.count >= currentBatchSize
if shouldProcess {
Expand All @@ -413,7 +413,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
}

/// Process a batch of executions
private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose)]) async {
private func processExecutionBatch(_ batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)]) async {
guard let fuzzerId = fuzzerId else {
logger.error("Cannot process execution batch: fuzzer not registered")
return
Expand All @@ -428,7 +428,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
var uniquePrograms: [String: (Program, ExecutionMetadata)] = [:]
var executionBatchData: [ExecutionBatchData] = []

for (program, aspects, executionType) in batch {
for (program, aspects, executionType, execData) in batch {
// Filter out test programs with FUZZILLI_CRASH (false positive crashes)
if DatabaseUtils.containsFuzzilliCrash(program: program) {
if enableLogging {
Expand Down Expand Up @@ -464,7 +464,11 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
mutatorType: nil,
outcome: aspects.outcome,
coverage: coveragePct,
coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set<Int>()
executionTimeMs: Int(execData.execTime * 1000),
coverageEdges: (aspects as? CovEdgeSet).map { Set($0.getEdges().map { Int($0) }) } ?? Set<Int>(),
stdout: execData.stdout,
stderr: execData.stderr,
fuzzout: execData.fuzzout
)
executionBatchData.append(executionData)
}
Expand All @@ -491,7 +495,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {

/// Flush any pending executions in the batch
private func flushExecutionBatch() {
let batch = executionBatchLock.withLock {
let batch: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
let batch = pendingExecutions
pendingExecutions.removeAll()
return batch
Expand Down Expand Up @@ -521,7 +525,7 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
}

// Process any queued executions
let queuedExecutions = executionBatchLock.withLock {
let queuedExecutions: [(Program, ProgramAspects, DatabaseExecutionPurpose, ExecutionData)] = executionBatchLock.withLock {
let queuedExecutions = pendingExecutions
pendingExecutions.removeAll()
return queuedExecutions
Expand Down Expand Up @@ -681,53 +685,15 @@ public class PostgreSQLCorpus: ComponentBase, Corpus {
return
}

do {
// Use the registered fuzzer ID
guard let fuzzerId = fuzzerId else {
if enableLogging {
self.logger.info("Cannot store execution: fuzzer not registered")
}
return
}

// DEBUG: Log execution storage attempt
if enableLogging {
self.logger.info("Storing execution: fuzzerId=\(fuzzerId), outcome=\(executionData.outcome), execTime=\(executionData.execTime)")
}

// Derive coverage percentage (0-100) from evaluator if available
let coveragePct: Double = {
if let coverageEvaluator = self.fuzzer.evaluator as? ProgramCoverageEvaluator {
return coverageEvaluator.currentScore * 100.0
} else {
return 0.0
}
}()

// Store both program and execution in a single transaction to avoid foreign key issues
_ = try await storage.storeProgramAndExecution(
program: program,
fuzzerId: fuzzerId,
executionType: executionType,
outcome: executionData.outcome,
coverage: coveragePct,
executionTimeMs: Int(executionData.execTime * 1000),
stdout: executionData.stdout,
stderr: executionData.stderr,
fuzzout: executionData.fuzzout,
metadata: ExecutionMetadata(lastOutcome: DatabaseExecutionOutcome(
id: DatabaseUtils.mapExecutionOutcome(outcome: aspects.outcome),
outcome: aspects.outcome.description,
description: aspects.outcome.description
))
)

if enableLogging {
self.logger.info("Successfully stored program and execution")
// Add to execution batch for batched storage (even if fuzzer not registered yet)
addToExecutionBatch(program, aspects, executionType: executionType, executionData: executionData)

if enableLogging {
if fuzzerId == nil {
self.logger.info("Queued execution for later (fuzzer not registered yet)")
} else {
self.logger.info("Added execution to batch: outcome=\(executionData.outcome), execTime=\(executionData.execTime)")
}

} catch {
logger.error("Failed to store execution: \(String(reflecting: error))")
}
}

Expand Down
Loading