test_format_validation.py

#!/usr/bin/env python3
"""
Test Format Validation - Validates result.csv and traces.json formatting
Critical for Phase 1 baseline submission success
"""

import os
import sys
import json
import logging
from pathlib import Path

# Add project root to path
sys.path.insert(0, os.path.dirname(__file__))

from utils.format_validator import get_format_validator
from agent.trace_logger import TraceLogger

# Configure logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def test_format_validation():
    """Test the complete format validation pipeline"""
    
    print("=== Testing Format Validation Pipeline ===\n")
    
    # Initialize validator
    validator = get_format_validator()
    
    # Test 1: Create sample submission
    print("1. Creating sample submission...")
    if validator.create_sample_submission("test_result.csv"):
        print("   Sample submission created successfully")
    else:
        print("   Failed to create sample submission")
        return False
    
    # Test 2: Create sample traces
    print("2. Creating sample traces...")
    if validator.create_sample_traces("test_traces.json"):
        print("   Sample traces created successfully")
    else:
        print("   Failed to create sample traces")
        return False
    
    # Test 3: Validate CSV format
    print("3. Validating CSV format...")
    csv_valid, csv_result = validator.validate_csv_format("test_result.csv")
    print(f"   CSV validation: {'PASS' if csv_valid else 'FAIL'}")
    if csv_valid:
        print(f"   Total rows: {csv_result['total_rows']}")
        print(f"   Unique IDs: {csv_result['unique_ids']}")
        print(f"   Track A non-empty: {csv_result['track_a_non_empty']}")
        print(f"   Track B non-empty: {csv_result['track_b_non_empty']}")
    else:
        print(f"   Error: {csv_result['error']}")
    
    # Test 4: Validate traces format
    print("4. Validating traces format...")
    traces_valid, traces_result = validator.validate_traces_format("test_traces.json")
    print(f"   Traces validation: {'PASS' if traces_valid else 'FAIL'}")
    if traces_valid:
        print(f"   Total entries: {traces_result['total_entries']}")
        print(f"   Unique actions: {traces_result['unique_actions']}")
        print(f"   Avg execution time: {traces_result['avg_execution_time']:.2f}s")
    else:
        print(f"   Error: {traces_result['error']}")
    
    # Test 5: Overall validation
    print("5. Overall validation...")
    overall_valid, overall_result = validator.validate_submission_package("test_result.csv", "test_traces.json")
    print(f"   Overall validation: {overall_result['overall_status']}")
    
    # Test 6: Test trace logger
    print("6. Testing trace logger...")
    trace_logger = TraceLogger("test_trace_logger.json")
    
    # Add a test trace entry
    trace_logger.add_trace_entry(
        iteration=0,
        thought="Test thought for validation",
        action="get_node_status",
        action_input={"node_id": "test-node"},
        observation={"status": "success", "node_state": "online"},
        execution_time=0.5
    )
    
    # Validate the trace logger output
    traces_valid, traces_result = validator.validate_traces_format("test_trace_logger.json")
    print(f"   Trace logger validation: {'PASS' if traces_valid else 'FAIL'}")
    
    # Test 7: Test edge cases
    print("7. Testing edge cases...")
    
    # Test empty Track B
    edge_case_data = [
        {"ID": "edge_001", "Track A": "C1|C2", "Track B": ""},
        {"ID": "edge_002", "Track A": "C3", "Track B": None},  # This should be converted to ""
    ]
    
    import pandas as pd
    df_edge = pd.DataFrame(edge_case_data)
    df_edge.to_csv("test_edge_case.csv", index=False)
    
    edge_valid, edge_result = validator.validate_csv_format("test_edge_case.csv")
    print(f"   Edge case validation: {'PASS' if edge_valid else 'FAIL'}")
    
    # Cleanup
    print("8. Cleaning up test files...")
    test_files = [
        "test_result.csv", "test_traces.json", "test_trace_logger.json",
        "test_edge_case.csv"
    ]
    
    for file in test_files:
        if os.path.exists(file):
            os.remove(file)
            print(f"   Removed: {file}")
    
    # Summary
    print("\n=== Test Summary ===")
    print(f"CSV Validation: {'PASS' if csv_valid else 'FAIL'}")
    print(f"Traces Validation: {'PASS' if traces_valid else 'FAIL'}")
    print(f"Overall Validation: {'PASS' if overall_valid else 'FAIL'}")
    print(f"Edge Case Validation: {'PASS' if edge_valid else 'FAIL'}")
    
    all_passed = csv_valid and traces_valid and overall_valid and edge_valid
    print(f"\nAll tests passed: {'YES' if all_passed else 'NO'}")
    
    return all_passed

def test_submission_format_compliance():
    """Test compliance with Zindi submission format"""
    
    print("\n=== Testing Zindi Submission Format Compliance ===\n")
    
    # Create a sample that matches Zindi requirements
    zindi_sample = [
        {"ID": "80e3aa96-815d-4683-980c-16db42eab0ef", "Track A": "C8|C11", "Track B": ""},
        {"ID": "f55a819f-3fb9-4c8f-8859-a5b1649ff2d5", "Track A": "C15|C21", "Track B": ""},
        {"ID": "2cd1f674-a411-4860-82c6-a16ba624b172", "Track A": "C10|C11", "Track B": ""},
    ]
    
    import pandas as pd
    df = pd.DataFrame(zindi_sample)
    df.to_csv("zindi_format_test.csv", index=False)
    
    # Validate
    validator = get_format_validator()
    is_valid, result = validator.validate_csv_format("zindi_format_test.csv")
    
    print(f"Zindi format compliance: {'PASS' if is_valid else 'FAIL'}")
    if is_valid:
        print("Format matches Zindi requirements:")
        print(f"- Total entries: {result['total_rows']}")
        print(f"- Unique IDs: {result['unique_ids']}")
        print(f"- Track A format: Valid")
        print(f"- Track B format: Valid (empty allowed)")
    
    # Cleanup
    os.remove("zindi_format_test.csv")
    
    return is_valid

if __name__ == "__main__":
    print("Starting format validation tests...\n")
    
    # Run all tests
    format_tests_passed = test_format_validation()
    zindi_compliance_passed = test_submission_format_compliance()
    
    # Final summary
    print("\n" + "="*50)
    print("FINAL TEST RESULTS")
    print("="*50)
    print(f"Format Validation Tests: {'PASS' if format_tests_passed else 'FAIL'}")
    print(f"Zindi Compliance Tests: {'PASS' if zindi_compliance_passed else 'FAIL'}")
    
    overall_success = format_tests_passed and zindi_compliance_passed
    print(f"\nOVERALL: {'READY FOR SUBMISSION' if overall_success else 'NEEDS FIXES'}")
    
    if overall_success:
        print("\nYour output formatting is ready for Phase 1 submission!")
        print("Both result.csv and traces.json will parse correctly on Zindi.")
    else:
        print("\nPlease fix the formatting issues before submitting.")
    
    exit(0 if overall_success else 1)