vrp_evaluator.py

import argparse
import json
import sys
import os
import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

# ==============================================================================
# 1. GRAPH & ROUTE UTILITIES
# ==============================================================================

def load_graph(path):
    """Loads the graph.json into a NetworkX DiGraph with weights."""
    with open(path) as f:
        data = json.load(f)
    
    G = nx.DiGraph()
    for n in data['nodes']:
        G.add_node(n['id'], lat=n['lat'], lon=n['lon'])
        
    for e in data['edges']:
        w = e.get('average_time', 1.0)
        G.add_edge(e['u'], e['v'], weight=w)
        if not e.get('oneway', True):
            G.add_edge(e['v'], e['u'], weight=w)
            
    return G

def get_shortest_path_cost(G, u, v, cache):
    """Returns shortest path cost between u and v using a cache."""
    if u == v: 
        return 0.0
    if u not in cache:
        # Compute single-source shortest paths from u
        try:
            cache[u] = nx.single_source_dijkstra_path_length(G, u, weight='weight')
        except:
            cache[u] = {}
    return cache[u].get(v, float('inf'))

def analyze_queries(G, queries_path):
    """Identifies impossible orders (unreachable) in the query set."""
    with open(queries_path) as f:
        q_data = json.load(f)
    
    events = q_data['events']
    analysis = []
    
    # Pre-compute reachability cache
    # For small graph (50 nodes), we can compute APSP or lazy load
    dist_cache = {}
    
    for i, ev in enumerate(events):
        depot = ev['fleet']['depot_node']
        orders = ev['orders']
        
        impossible_ids = set()
        serviceable_ids = set()
        
        for o in orders:
            oid = o['order_id']
            p = o['pickup']
            d = o['dropoff']
            
            # Check Depot -> P
            dist_dp = get_shortest_path_cost(G, depot, p, dist_cache)
            # Check P -> D
            dist_pd = get_shortest_path_cost(G, p, d, dist_cache)
            
            if dist_dp == float('inf') or dist_pd == float('inf'):
                impossible_ids.add(oid)
            else:
                serviceable_ids.add(oid)
                
        analysis.append({
            "event_idx": i,
            "total_orders": len(orders),
            "impossible": impossible_ids,
            "serviceable": serviceable_ids,
            "orders_map": {o['order_id']: o for o in orders},
            "depot": depot
        })
        
    return analysis, dist_cache

# ==============================================================================
# 2. VALIDATION LOGIC
# ==============================================================================

def validate_solution(G, event_analysis, result_event, dist_cache):
    """
    Validates a single event solution.
    Returns metrics and validity flags.
    """
    depot = event_analysis['depot']
    orders_map = event_analysis['orders_map']
    
    assignments = result_event.get('assignments', [])
    metrics = result_event.get('metrics', {})
    reported_cost = metrics.get('total_delivery_time_s', 0)
    
    calculated_score = 0.0
    assigned_order_ids = set()
    
    is_valid = True
    validation_errors = []
    
    # 1. Evaluate each driver's route
    for driver in assignments:
        route = driver['route'] # List of Physical Node IDs
        driver_orders = driver['order_ids']
        
        # Check A: Route must start at depot
        if not route or route[0] != depot:
            is_valid = False
            validation_errors.append(f"Driver {driver.get('driver_id')} route does not start at depot.")
            continue

        # Check B: Connectivity and Timeline
        current_time = 0.0
        curr_node = route[0]
        
        # Tracking delivery status for this driver
        picked_up = set()
        delivered = set()
        
        # We need to map which node corresponds to P or D for the assigned orders
        # Note: One node might be P for Order A and D for Order B
        
        for next_node in route[1:]:
            # Move
            cost = get_shortest_path_cost(G, curr_node, next_node, dist_cache)
            if cost == float('inf'):
                is_valid = False
                validation_errors.append(f"Driver {driver.get('driver_id')} path disconnected: {curr_node} -> {next_node}")
                break
            
            current_time += cost
            curr_node = next_node
            
            # Process Actions at this node for ASSIGNED orders
            for oid in driver_orders:
                o = orders_map[oid]
                
                # Pickup?
                if o['pickup'] == curr_node and oid not in picked_up:
                    picked_up.add(oid)
                
                # Dropoff?
                if o['dropoff'] == curr_node:
                    # Check Precedence
                    if oid in picked_up:
                        if oid not in delivered:
                            delivered.add(oid)
                            calculated_score += current_time # Accumulate Arrival Time
                    # If dropped but not picked up yet -> Violation (unless P and D are same node and handled instantly)
                    elif oid not in delivered:
                         # If P == D == curr_node, it's valid.
                         if o['pickup'] == curr_node:
                             picked_up.add(oid) # Picked up just now
                             delivered.add(oid)
                             calculated_score += current_time
                         else:
                             is_valid = False
                             validation_errors.append(f"Order {oid} dropped at {curr_node} before pickup.")

        assigned_order_ids.update(delivered)
    
    # 2. Calculate Penalties
    all_orders = set(orders_map.keys())
    unassigned = all_orders - assigned_order_ids
    penalty = len(unassigned) * 1e9
    total_calculated_cost = calculated_score + penalty
    
    # 3. Metrics
    stats = {
        "is_valid": is_valid,
        "reported_cost": reported_cost,
        "calculated_cost": total_calculated_cost,
        "cost_diff": abs(reported_cost - total_calculated_cost),
        "assigned_count": len(assigned_order_ids),
        "unassigned_count": len(unassigned),
        "impossible_served": len(assigned_order_ids.intersection(event_analysis['impossible'])),
        "serviceable_missed": len(event_analysis['serviceable'] - assigned_order_ids),
        "errors": validation_errors
    }
    
    return stats

# ==============================================================================
# 3. MAIN SCRIPT
# ==============================================================================

def main():
    parser = argparse.ArgumentParser(description="Compare VRP Solver Results")
    parser.add_argument("graph", help="Path to graph.json")
    parser.add_argument("queries", help="Path to queries.json")
    parser.add_argument("outputs", nargs='+', help="List of output JSON files to compare (e.g. output_lib.json output_my.json)")
    args = parser.parse_args()

    # Load Data
    print(f"Loading graph: {args.graph}")
    G = load_graph(args.graph)
    
    print(f"Analyzing queries: {args.queries}")
    query_analysis, dist_cache = analyze_queries(G, args.queries)
    
    results_db = {} # Filename -> [EventStats]
    
    # Process each output file
    for out_file in args.outputs:
        if not os.path.exists(out_file):
            print(f"Warning: File {out_file} not found. Skipping.")
            continue
            
        print(f"Processing {out_file}...")
        with open(out_file) as f:
            try:
                res_json = json.load(f)
            except json.JSONDecodeError:
                print(f"  Error: Invalid JSON in {out_file}")
                continue
        
        file_stats = []
        results_list = res_json.get('results', [])
        
        for i, ev_res in enumerate(results_list):
            if i >= len(query_analysis): break
            
            stats = validate_solution(G, query_analysis[i], ev_res, dist_cache)
            file_stats.append(stats)
            
            # Sanity Print
            status = "VALID" if stats['is_valid'] else "INVALID"
            print(f"  Event {i}: {status} | Reported: {stats['reported_cost']:.2e} | Calc: {stats['calculated_cost']:.2e} | Orders: {stats['assigned_count']}/{query_analysis[i]['total_orders']}")
            if not stats['is_valid']:
                for e in stats['errors'][:2]: print(f"    - {e}")
        
        results_db[os.path.basename(out_file)] = file_stats

    # ==========================================================================
    # 4. PLOTTING
    # ==========================================================================
    
    if not results_db:
        print("No valid results to plot.")
        return

    events = range(len(query_analysis))
    filenames = list(results_db.keys())
    x = np.arange(len(events))
    width = 0.8 / len(filenames)
    
    # PLOT 1: Accuracy (Orders Served)
    fig, ax = plt.subplots(figsize=(12, 6))
    
    for i, fname in enumerate(filenames):
        # Prepare data
        assigned = [results_db[fname][j]['assigned_count'] for j in events]
        # We assume impossible are never served, so max possible is serviceable
        
        offset = x + i * width
        ax.bar(offset, assigned, width, label=fname)
        
    # Add line for Max Serviceable
    serviceable_counts = [len(qa['serviceable']) for qa in query_analysis]
    ax.plot(x + width * (len(filenames)-1)/2, serviceable_counts, 'r--', marker='o', label='Max Serviceable (Theoretical)')
    
    ax.set_ylabel('Orders Delivered')
    ax.set_xlabel('Query Event ID')
    ax.set_title('Accuracy Comparison: Orders Delivered vs Serviceable')
    ax.set_xticks(x + width * (len(filenames)-1)/2)
    ax.set_xticklabels([f"Q{j}" for j in events])
    ax.legend()
    plt.grid(axis='y', alpha=0.3)
    plt.savefig('accuracy_comparison.png')
    print("Saved accuracy_comparison.png")

    # PLOT 2: Quality (Total Cost)
    fig2, ax2 = plt.subplots(figsize=(12, 6))
    
    for i, fname in enumerate(filenames):
        costs = [results_db[fname][j]['calculated_cost'] for j in events]
        offset = x + i * width
        ax2.bar(offset, costs, width, label=fname)
        
    ax2.set_ylabel('Total Cost (Log Scale)')
    ax2.set_xlabel('Query Event ID')
    ax2.set_title('Quality Comparison: Objective Function (Lower is Better)')
    ax2.set_yscale('log') # Log scale because penalties are 1e9
    ax2.set_xticks(x + width * (len(filenames)-1)/2)
    ax2.set_xticklabels([f"Q{j}" for j in events])
    ax2.legend()
    plt.grid(axis='y', alpha=0.3)
    plt.savefig('quality_comparison.png')
    print("Saved quality_comparison.png")
    
    # Summary Table
    print("\n=== Summary Discrepancies (Calc vs Reported) ===")
    for fname in filenames:
        diffs = [s['cost_diff'] for s in results_db[fname]]
        avg_diff = sum(diffs)/len(diffs) if diffs else 0
        valid_count = sum(1 for s in results_db[fname] if s['is_valid'])
        print(f"{fname}: {valid_count}/{len(events)} valid events. Avg Cost Discrepancy: {avg_diff:.2f}")

if __name__ == "__main__":
    main()