test_sdk_build.py

#!/usr/bin/env python3
"""
Test: Build Analytics SDK with adversarial validation vs baseline
"""
import sys
from pathlib import Path
from adversarial_coding import adversarial_build, build_with_baseline
import json

# Test task
SDK_TASK = """
Build a JavaScript Analytics SDK for event tracking.

REQUIREMENTS:
- Batching: Queue events and send in batches (configurable batch size)
- Retry logic: Retry failed requests with exponential backoff
- Validation: Reject malformed events before queuing
- Rate limiting: Prevent overwhelming the API
- Offline support: Queue events when offline, send when back online
- Tests: Comprehensive unit tests for all functionality
- Health check: Script that validates SDK is working correctly

SCALE: Must efficiently handle 1M events/day (batching is critical)

OUTPUT:
- sdk.js (or sdk/ directory with modules)
- tests/ directory with test files
- health-check.js script
- README.md with usage examples
- package.json with dependencies
"""

def run_test():
    """Run the validation test"""
    print("\n" + "="*70)
    print("🧪 VALIDATION TEST: Analytics SDK")
    print("="*70)
    
    base_dir = Path(__file__).parent
    
    # Test 1: Baseline (single agent)
    print("\n\n📦 TEST 1: BASELINE BUILD (Single Agent)")
    print("="*70)
    baseline_dir = base_dir / "test_output" / "baseline_sdk"
    baseline_dir.mkdir(parents=True, exist_ok=True)
    
    try:
        baseline_result = build_with_baseline(
            SDK_TASK,
            str(baseline_dir),
            agent_type="codex",
            verbose=True
        )
    except Exception as e:
        print(f"❌ Baseline build failed with error: {e}")
        baseline_result = {"success": False, "error": str(e)}
    
    # Test 2: Adversarial
    print("\n\n⚔️  TEST 2: ADVERSARIAL BUILD")
    print("="*70)
    adversarial_dir = base_dir / "test_output" / "adversarial_sdk"
    adversarial_dir.mkdir(parents=True, exist_ok=True)
    
    try:
        adversarial_result = adversarial_build(
            SDK_TASK,
            str(adversarial_dir),
            max_rounds=3,
            agent_type="codex",
            verbose=True
        )
    except Exception as e:
        print(f"❌ Adversarial build failed with error: {e}")
        adversarial_result = {"success": False, "error": str(e)}
    
    # Compare results
    print("\n\n" + "="*70)
    print("📊 COMPARISON")
    print("="*70)
    
    comparison = {
        "baseline": {
            "success": baseline_result.get("success", False),
            "rounds": baseline_result.get("rounds", 0),
            "files_created": len(baseline_result.get("files_created", [])),
            "tests_exist": baseline_result.get("validation", {}).get("tests_exist", False),
            "tests_pass": baseline_result.get("validation", {}).get("tests_pass", False),
            "workdir": str(baseline_dir)
        },
        "adversarial": {
            "success": adversarial_result.get("success", False),
            "rounds": adversarial_result.get("rounds", 0),
            "files_created": len(adversarial_result.get("files_created", [])),
            "tests_exist": adversarial_result.get("validation", {}).get("tests_exist", False),
            "tests_pass": adversarial_result.get("validation", {}).get("tests_pass", False),
            "workdir": str(adversarial_dir)
        }
    }
    
    print("\nBASELINE:")
    print(f"  Success: {comparison['baseline']['success']}")
    print(f"  Files: {comparison['baseline']['files_created']}")
    print(f"  Tests exist: {comparison['baseline']['tests_exist']}")
    print(f"  Tests pass: {comparison['baseline']['tests_pass']}")
    
    print("\nADVERSARIAL:")
    print(f"  Success: {comparison['adversarial']['success']}")
    print(f"  Rounds: {comparison['adversarial']['rounds']}")
    print(f"  Files: {comparison['adversarial']['files_created']}")
    print(f"  Tests exist: {comparison['adversarial']['tests_exist']}")
    print(f"  Tests pass: {comparison['adversarial']['tests_pass']}")
    
    print("\nWINNER:")
    if comparison['adversarial']['success'] and not comparison['baseline']['success']:
        print("  ⚔️  ADVERSARIAL - Only adversarial produced working code")
    elif comparison['baseline']['success'] and not comparison['adversarial']['success']:
        print("  📦 BASELINE - Only baseline produced working code")
    elif comparison['adversarial']['success'] and comparison['baseline']['success']:
        print("  🤝 BOTH - Need to compare quality manually")
    else:
        print("  ❌ NEITHER - Both failed to produce working code")
    
    print(f"\n📂 Check outputs:")
    print(f"  Baseline: {baseline_dir}")
    print(f"  Adversarial: {adversarial_dir}")
    
    # Save comparison
    comparison_file = base_dir / "test_output" / "comparison.json"
    with open(comparison_file, 'w') as f:
        json.dump(comparison, f, indent=2)
    print(f"\n📊 Comparison saved to: {comparison_file}")
    
    return comparison

if __name__ == "__main__":
    comparison = run_test()
    
    # Exit code: 0 if adversarial is better or equal, 1 otherwise
    adversarial_better = (
        comparison['adversarial']['success'] and 
        (not comparison['baseline']['success'] or 
         comparison['adversarial']['tests_pass'])
    )
    
    sys.exit(0 if adversarial_better else 1)