optimize_system.py

#!/usr/bin/env python3
"""
System Optimization Script
=========================

This script automatically optimizes the RecognAIze system by:
1. Regenerating embeddings with improved feature extraction
2. Tuning similarity threshold on training data
3. Running optimized inference on test data

Usage:
    python optimize_system.py
"""

import subprocess
import sys
import os
import json
from pathlib import Path

def run_command(cmd, description):
    """Run a command and print progress."""
    print(f"\n{'='*60}")
    print(f"{description}")
    print('='*60)
    print(f"Running: {' '.join(cmd)}")
    
    try:
        result = subprocess.run(cmd, check=True, capture_output=False)
        print(f"✅ {description} completed successfully!")
        return True
    except subprocess.CalledProcessError as e:
        print(f"❌ {description} failed with exit code {e.returncode}")
        return False

def main():
    """Main optimization workflow."""
    print("🚀 RecognAIze System Optimization")
    print("This will optimize the system for maximum accuracy!")
    
    # Step 1: Clean up old embeddings to force regeneration
    print("\n🧹 Cleaning up old embeddings...")
    embeddings_dir = Path("embeddings")
    if embeddings_dir.exists():
        import shutil
        shutil.rmtree(embeddings_dir)
        print("✅ Old embeddings removed")
    
    # Step 2: Run training with threshold optimization
    if not run_command([
        sys.executable, "main.py", "train",
        "--dataset", "identity-employees-in-surveillance-cctv/dataset",
        "--confidence", "0.2",
        "--debug"
    ], "Training with threshold optimization"):
        return False
    
    # Step 3: Load the optimal threshold
    optimal_threshold = 0.85  # Default fallback
    config_path = "optimal_config.json"
    
    if os.path.exists(config_path):
        try:
            with open(config_path, 'r') as f:
                config = json.load(f)
            optimal_threshold = config.get('optimal_similarity_threshold', 0.85)
            print(f"✅ Loaded optimal threshold: {optimal_threshold:.3f}")
        except Exception as e:
            print(f"⚠️ Could not load optimal config: {e}")
    
    # Step 4: Run inference with optimized parameters
    if not run_command([
        sys.executable, "main.py", "inference",
        "--dataset", "identity-employees-in-surveillance-cctv/dataset", 
        "--threshold", str(optimal_threshold),
        "--confidence", "0.3",
        "--debug"
    ], "Running optimized inference"):
        return False
    
    # Step 5: Show results summary
    print(f"\n{'='*60}")
    print("🎉 OPTIMIZATION COMPLETE!")
    print('='*60)
    
    # Try to show some stats
    results_file = Path("results/submission.csv")
    if results_file.exists():
        with open(results_file, 'r') as f:
            lines = f.readlines()
        
        total_predictions = len(lines) - 1  # Exclude header
        unknown_count = sum(1 for line in lines[1:] if 'UNKNOWN' in line)
        identified_count = total_predictions - unknown_count
        
        print(f"📊 Results Summary:")
        print(f"  Total predictions: {total_predictions}")
        print(f"  Employees identified: {identified_count} ({100*identified_count/total_predictions:.1f}%)")
        print(f"  Unknown faces: {unknown_count} ({100*unknown_count/total_predictions:.1f}%)")
        print(f"  Optimal threshold used: {optimal_threshold:.3f}")
        
        if unknown_count == 0:
            print("⚠️  WARNING: No UNKNOWN predictions - threshold might be too low!")
        elif unknown_count > total_predictions * 0.5:
            print("⚠️  WARNING: Too many UNKNOWN predictions - threshold might be too high!")
        else:
            print("✅ Prediction distribution looks reasonable!")
    
    print(f"\n📁 Output files:")
    print(f"  results/submission.csv")
    print(f"  results/inference_stats.txt")
    print(f"  optimal_config.json")
    
    return True

if __name__ == "__main__":
    success = main()
    if success:
        print(f"\n🎯 System optimization completed! Check results/submission.csv")
        sys.exit(0)
    else:
        print(f"\n💥 System optimization failed!")
        sys.exit(1)