evaluate_arc.py

#!/usr/bin/env python3
"""
ARC Challenge Evaluation with High-Performance BIOMIND
====================================================

Direct evaluation of BIOMIND system on ARC (AI2 Reasoning Challenge).
"""

import sys
import time
import json
import random
import numpy as np
from pathlib import Path
from typing import Dict, List, Any, Optional, Tuple

# Add project root to Python path  
sys.path.insert(0, str(Path(__file__).parent / ".cleanup_backup"))

# Import working BIOMIND system
from integrated_biomind_evaluation import IntegratedBIOMINDEvaluator

def load_arc_samples(num_samples: int = 50) -> List[Dict[str, Any]]:
    """Load ARC challenge questions."""
    
    # Sample ARC-Challenge questions (grade-school science reasoning)
    arc_samples = [
        {
            "question": "A student wants to know which type of soil is best for growing plants. What should the student do?",
            "choices": {
                "A": "Plant seeds in different types of soil and see which grows best",  # Correct
                "B": "Ask a teacher which soil is best",
                "C": "Look up the answer in a book", 
                "D": "Use the most expensive soil"
            },
            "answer": "A",
            "subject": "experimental_design"
        },
        {
            "question": "What happens to most of the water that falls on land as precipitation?",
            "choices": {
                "A": "It becomes groundwater",
                "B": "It evaporates back into the atmosphere",  # Correct
                "C": "It flows to the ocean", 
                "D": "It freezes into ice"
            },
            "answer": "B", 
            "subject": "earth_science"
        },
        {
            "question": "Which tool would be most helpful for a student to measure the volume of a liquid?",
            "choices": {
                "A": "Balance scale",
                "B": "Thermometer",
                "C": "Graduated cylinder",  # Correct
                "D": "Magnifying glass"
            },
            "answer": "C",
            "subject": "measurement"
        },
        {
            "question": "A scientist observes that a plant always bends toward a window. What is the scientist most likely studying?",
            "choices": {
                "A": "How plants respond to light",  # Correct  
                "B": "How plants absorb water",
                "C": "How plants produce oxygen",
                "D": "How plants reproduce"
            },
            "answer": "A",
            "subject": "plant_biology"
        },
        {
            "question": "Which of these changes is most likely due to chemical weathering?",
            "choices": {
                "A": "A rock splits due to ice forming in its cracks",
                "B": "A statue made of limestone becomes pitted due to acid rain",  # Correct
                "C": "A mountainside is worn away by a glacier",
                "D": "A rock is broken into smaller pieces by wind"
            },
            "answer": "B",
            "subject": "geology"
        },
        {
            "question": "What is the main source of energy for the water cycle?",
            "choices": {
                "A": "Wind",
                "B": "The Sun",  # Correct
                "C": "Ocean currents",
                "D": "Underground heat"
            },
            "answer": "B",
            "subject": "earth_science"
        },
        {
            "question": "A student puts a drop of food coloring in a glass of hot water and a glass of cold water. In which glass will the food coloring spread faster?",
            "choices": {
                "A": "The cold water",
                "B": "The hot water",  # Correct
                "C": "Both will be the same",
                "D": "Neither will spread"
            },
            "answer": "B",
            "subject": "physics"
        },
        {
            "question": "Which characteristic do all living things share?",
            "choices": {
                "A": "They can move from place to place",
                "B": "They need sunlight to survive", 
                "C": "They are made of cells",  # Correct
                "D": "They live in water"
            },
            "answer": "C",
            "subject": "biology"
        },
        {
            "question": "What causes day and night on Earth?",
            "choices": {
                "A": "The Earth revolving around the Sun",
                "B": "The Earth rotating on its axis",  # Correct
                "C": "The Moon blocking the Sun's light",
                "D": "Clouds covering the Sun"
            },
            "answer": "B", 
            "subject": "astronomy"
        },
        {
            "question": "A student wants to separate a mixture of sand and salt. Which method would work best?",
            "choices": {
                "A": "Use a magnet",
                "B": "Add water and filter",  # Correct
                "C": "Heat the mixture", 
                "D": "Shake the mixture"
            },
            "answer": "B",
            "subject": "chemistry"
        },
        {
            "question": "Which of these is an example of a physical change?",
            "choices": {
                "A": "Wood burning to form ash",
                "B": "Iron rusting when exposed to air",
                "C": "Ice melting to form water",  # Correct
                "D": "Milk souring when left out"
            },
            "answer": "C",
            "subject": "chemistry"
        },
        {
            "question": "What do all food chains start with?",
            "choices": {
                "A": "Carnivores",
                "B": "Herbivores", 
                "C": "Producers",  # Correct
                "D": "Decomposers"
            },
            "answer": "C",
            "subject": "ecology"
        },
        {
            "question": "Which gas makes up the largest percentage of Earth's atmosphere?",
            "choices": {
                "A": "Oxygen",
                "B": "Carbon dioxide",
                "C": "Nitrogen",  # Correct
                "D": "Water vapor"
            },
            "answer": "C",
            "subject": "earth_science"
        },
        {
            "question": "A student notices that plants in a greenhouse grow faster than plants outside. What is the most likely reason?",
            "choices": {
                "A": "The greenhouse plants get more carbon dioxide",
                "B": "The greenhouse plants are protected from wind", 
                "C": "The greenhouse plants receive more controlled conditions",  # Correct
                "D": "The greenhouse plants need less water"
            },
            "answer": "C",
            "subject": "plant_biology"
        },
        {
            "question": "Which type of rock is formed by cooling and hardening of magma?",
            "choices": {
                "A": "Sedimentary",
                "B": "Metamorphic",
                "C": "Igneous",  # Correct
                "D": "Crystalline"
            },
            "answer": "C",
            "subject": "geology"
        },
        {
            "question": "What happens to the kinetic energy of molecules as temperature increases?",
            "choices": {
                "A": "It decreases",
                "B": "It increases",  # Correct
                "C": "It stays the same",
                "D": "It becomes potential energy"
            },
            "answer": "B",
            "subject": "physics"
        },
        {
            "question": "Which part of a plant is primarily responsible for photosynthesis?",
            "choices": {
                "A": "Roots",
                "B": "Stem",
                "C": "Leaves",  # Correct
                "D": "Flowers"
            },
            "answer": "C",
            "subject": "plant_biology"
        },
        {
            "question": "What is the correct order of the phases of the moon?",
            "choices": {
                "A": "New, full, first quarter, last quarter",
                "B": "New, first quarter, full, last quarter",  # Correct
                "C": "Full, new, first quarter, last quarter",
                "D": "First quarter, full, last quarter, new"
            },
            "answer": "B",
            "subject": "astronomy"
        },
        {
            "question": "Which scientist is famous for developing the theory of evolution by natural selection?",
            "choices": {
                "A": "Albert Einstein",
                "B": "Isaac Newton",
                "C": "Charles Darwin",  # Correct
                "D": "Gregor Mendel"
            },
            "answer": "C",
            "subject": "biology"
        },
        {
            "question": "What is the smallest unit of matter that retains the properties of an element?",
            "choices": {
                "A": "Molecule",
                "B": "Atom",  # Correct
                "C": "Electron",
                "D": "Proton"
            },
            "answer": "B",
            "subject": "chemistry"
        }
    ]
    
    # Extend with more samples if needed by cycling through
    if num_samples > len(arc_samples):
        extended_samples = arc_samples * ((num_samples // len(arc_samples)) + 1)
        return extended_samples[:num_samples]
    
    return arc_samples[:num_samples]


def evaluate_arc_performance(evaluator: IntegratedBIOMINDEvaluator, samples: List[Dict], max_samples: int = None) -> Dict[str, Any]:
    """Evaluate BIOMIND on ARC questions."""
    
    if max_samples:
        samples = samples[:max_samples]
    
    print(f"\n[BRAIN] Starting ARC evaluation on {len(samples)} questions...")
    print("=" * 60)
    
    correct = 0
    total = len(samples)
    results = []
    subject_performance = {}
    processing_times = []
    
    for i, sample in enumerate(samples, 1):
        question = sample['question']
        choices = sample['choices'] 
        correct_answer = sample['answer']
        subject = sample['subject']
        
        # Format task for BIOMIND
        choices_text = "\n".join([f"{k}: {v}" for k, v in choices.items()])
        task = f"Question: {question}\n\nChoices:\n{choices_text}"
        
        print(f"\n[SEARCH] Question {i}/{total} ({subject})")
        print(f"Q: {question}")
        
        start_time = time.time()
        
        # Process with BIOMIND using correct method
        choices_list = list(choices.values())
        result = evaluator.evaluate_with_reflection(question, choices_list, correct_answer, subject)
        
        # Extract predicted answer from result
        predicted_answer = result.get('predicted_answer', '')
        processing_time = result.get('processing_time_ms', 0) / 1000  # Convert to seconds
        processing_times.append(processing_time)
        
        # Check if correct
        is_correct = result.get('is_correct', False)
        if is_correct:
            correct += 1
            
        # Track subject performance
        if subject not in subject_performance:
            subject_performance[subject] = {'correct': 0, 'total': 0}
        subject_performance[subject]['total'] += 1
        if is_correct:
            subject_performance[subject]['correct'] += 1
        
        print(f"   Predicted: {predicted_answer} | Correct: {correct_answer} | {'[OK]' if is_correct else '[FAIL]'}")
        print(f"   Time: {processing_time*1000:.1f}ms | Confidence: {result.get('confidence', 0):.3f}")
        
        results.append({
            'question': question,
            'choices': choices,
            'correct_answer': correct_answer,
            'predicted_answer': predicted_answer,
            'is_correct': is_correct,
            'confidence': result.get('confidence', 0),
            'processing_time_ms': processing_time * 1000,
            'subject': subject,
            'specialist': result.get('selected_specialist', 'unknown'),
            'routing_method': result.get('routing_method', 'unknown')
        })
        
        # Progress indicator
        current_accuracy = (correct / i) * 100
        print(f"   Progress: {i}/{total} ({i/total*100:.1f}%) | Running accuracy: {current_accuracy:.1f}%")
    
    # Calculate final metrics
    accuracy = (correct / total) * 100
    avg_processing_time = np.mean(processing_times) * 1000  # Convert to ms
    
    print(f"\n" + "=" * 60)
    print(f"[TARGET] ARC EVALUATION COMPLETE")
    print(f"=" * 60)
    print(f"[CHART] Overall Accuracy: {correct}/{total} ({accuracy:.1f}%)")
    print(f"?  Average Processing Time: {avg_processing_time:.1f}ms")
    print(f"[BRAIN] Total Questions: {total}")
    
    # Subject breakdown
    print(f"\n[DATA] Performance by Subject:")
    for subject, stats in subject_performance.items():
        subj_accuracy = (stats['correct'] / stats['total']) * 100
        print(f"   {subject.replace('_', ' ').title()}: {stats['correct']}/{stats['total']} ({subj_accuracy:.1f}%)")
    
    return {
        'total_questions': total,
        'correct_answers': correct,
        'accuracy_percent': accuracy,
        'average_processing_time_ms': avg_processing_time,
        'subject_performance': subject_performance,
        'detailed_results': results
    }


def main():
    """Run ARC evaluation."""
    
    print("[ROCKET] Initializing BIOMIND for ARC Challenge Evaluation")
    print("=" * 60)
    
    # Initialize BIOMIND evaluator with specialists
    specialists = ['qwen_math_expert', 'qwen_general_reasoner', 'tiny_llama_planner', 'tiny_llama_critic']
    evaluator = IntegratedBIOMINDEvaluator(specialists)
    print("[OK] BIOMIND system initialized")
    
    # Load ARC samples
    num_questions = 20  # Start with smaller set for testing
    samples = load_arc_samples(num_questions)
    print(f"[OK] Loaded {len(samples)} ARC questions")
    
    # Run evaluation
    results = evaluate_arc_performance(evaluator, samples, max_samples=num_questions)
    
    # Save results
    timestamp = time.strftime("%Y%m%d_%H%M%S")
    results_file = f"arc_evaluation_results_{timestamp}.json"
    
    with open(results_file, 'w') as f:
        json.dump(results, f, indent=2, default=str)
    
    print(f"\n? Results saved to: {results_file}")
    print(f"[TARGET] Final ARC Accuracy: {results['accuracy_percent']:.1f}%")
    

if __name__ == "__main__":
    main()