From a9220bdb78e1a6f054bee61efb176aaeb6be9ffd Mon Sep 17 00:00:00 2001 From: Leandro Echevarria Date: Thu, 21 Aug 2025 10:51:33 -0300 Subject: [PATCH 1/2] feat | LAY-907 Updated docs, added examples, updated tests for paginated results --- docs/examples/advanced-usage.md | 726 ++++++--------- docs/examples/creating-evaluations.md | 843 +++--------------- docs/examples/retrieving-results.md | 1135 +++--------------------- docs/examples/timeouts.md | 760 +++------------- examples/get_benchmarks.py | 25 + examples/get_evaluation.py | 19 + examples/get_models.py | 37 + examples/paginated_results.py | 105 +++ src/atlas/resources/results/results.py | 8 +- tests/resources/test_results.py | 16 +- 10 files changed, 840 insertions(+), 2834 deletions(-) create mode 100644 examples/get_benchmarks.py create mode 100644 examples/get_evaluation.py create mode 100644 examples/get_models.py create mode 100644 examples/paginated_results.py diff --git a/docs/examples/advanced-usage.md b/docs/examples/advanced-usage.md index d50383c..e880f7e 100644 --- a/docs/examples/advanced-usage.md +++ b/docs/examples/advanced-usage.md @@ -1,423 +1,303 @@ -# Advanced Usage Patterns +# Advanced Usage -This guide covers practical advanced techniques for using the Atlas Python SDK in production environments. +Common patterns and best practices for using the Atlas Python SDK in production. -## Environment Variables Setup - -The Atlas SDK reads your credentials from environment variables. You can set them up however you prefer: +## Environment Setup ```python -import os -from atlas import Atlas - -# Option 1: Load from system environment variables -client = Atlas() # Automatically uses LAYERLENS_ATLAS_API_KEY, etc. +# Set your API key as an environment variable +export LAYERLENS_ATLAS_API_KEY="your_api_key_here" -# Option 2: Using python-dotenv (if you prefer .env files) -from dotenv import load_dotenv -load_dotenv() # Loads from .env file -client = Atlas() +# Then in your code: +from atlas import Atlas +client = Atlas() # Automatically reads from environment ``` -Required environment variables: +## Async Operations -- `LAYERLENS_ATLAS_API_KEY` - Your Atlas API key - -## Pagination Best Practices - -### Understanding Pagination - -The Atlas SDK automatically handles pagination for large result sets. When evaluation results exceed the default page size (100), you'll need to iterate through pages to access all data. +### Basic Async Usage ```python -from atlas import Atlas - -def understand_pagination(evaluation_id: str): - """Understand pagination metadata""" - client = Atlas() - - # Get first page - results_data = client.results.get(evaluation_id=evaluation_id) - - if results_data: - pagination = results_data.pagination - - print(f" Pagination Overview:") - print(f" Total results: {pagination.total_count:,}") - print(f" Page size: {pagination.page_size}") - print(f" Total pages: {pagination.total_pages}") - print(f" Current page has: {len(results_data.results)} results") - - # Calculate some useful info - is_paginated = pagination.total_pages > 1 - results_per_page = pagination.page_size - last_page_size = pagination.total_count % pagination.page_size or pagination.page_size - - print(f"\n Analysis:") - print(f" Is paginated: {is_paginated}") - print(f" Results per page: {results_per_page}") - print(f" Last page size: {last_page_size}") - - if is_paginated: - print(f"\n To access all {pagination.total_count:,} results:") - print(f" - Iterate through {pagination.total_pages} pages") - print(f" - Or use batch processing patterns") - - return pagination - +import asyncio +from atlas import AsyncAtlas + +async def run_evaluation(): + # Create async client + client = AsyncAtlas() + + # Get models and benchmarks concurrently + models_task = client.models.get() + benchmarks_task = client.benchmarks.get() + models, benchmarks = await asyncio.gather(models_task, benchmarks_task) + + # Create evaluation + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] + ) + + # Wait for completion + completed = await client.evaluations.wait_for_completion(evaluation) + + if completed.is_success: + # Get results + results = await client.results.get_by_id(evaluation_id=completed.id) + return results + return None -# Usage -pagination_info = understand_pagination("eval_12345") +# Run it +results = asyncio.run(run_evaluation()) ``` -### Efficient Pagination Strategies +### Multiple Concurrent Evaluations ```python -def efficient_pagination_strategies(): - """Demonstrate different pagination approaches""" - client = Atlas() - evaluation_id = "eval_12345" - - # Strategy 1: Small pages for real-time processing - print(" Strategy 1: Small pages for real-time feedback") - page_size = 25 - page = 1 - - while True: - results_data = client.results.get( - evaluation_id=evaluation_id, - page=page, - page_size=page_size - ) - - if not results_data or not results_data.results: - break - - print(f" Processing page {page}: {len(results_data.results)} results") - - # Process immediately - for result in results_data.results: - # Real-time processing logic - pass - - if page >= results_data.pagination.total_pages: - break - page += 1 - - print("\n Strategy 2: Large pages for batch processing") - page_size = 200 # Larger pages - page = 1 - - while True: - results_data = client.results.get( - evaluation_id=evaluation_id, - page=page, - page_size=page_size - ) - - if not results_data or not results_data.results: - break - - print(f" Batch processing page {page}: {len(results_data.results)} results") - - # Batch process entire page - process_batch(results_data.results) - - if page >= results_data.pagination.total_pages: - break - page += 1 - -def process_batch(results): - """Process a batch of results efficiently""" - # Batch processing logic here - pass - -# Usage -efficient_pagination_strategies() +import asyncio +from atlas import AsyncAtlas + +async def create_multiple_evaluations(): + client = AsyncAtlas() + + models = await client.models.get() + benchmarks = await client.benchmarks.get() + + # Create multiple evaluations concurrently + tasks = [] + for model in models[:3]: # First 3 models + for benchmark in benchmarks[:2]: # First 2 benchmarks + task = client.evaluations.create(model=model, benchmark=benchmark) + tasks.append(task) + + # Wait for all to complete + evaluations = await asyncio.gather(*tasks) + + # Filter successful ones + successful = [e for e in evaluations if e is not None] + print(f"Created {len(successful)} evaluations") + + return successful + +# Run it +evaluations = asyncio.run(create_multiple_evaluations()) ``` -## Batch Processing +## Error Handling -### Running Multiple Evaluations +### Basic Error Handling ```python -import time from atlas import Atlas import atlas -def run_evaluation_batch(models, benchmarks): - """Run evaluations for multiple model-benchmark combinations""" +def safe_create_evaluation(): client = Atlas() + + try: + models = client.models.get() + benchmarks = client.benchmarks.get() + + evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] + ) + + return evaluation + + except atlas.AuthenticationError: + print("Invalid API key") + return None + except atlas.NotFoundError: + print("Model or benchmark not found") + return None + except atlas.RateLimitError as e: + print(f"Rate limited. Try again in {e.retry_after} seconds") + return None + except atlas.APIError as e: + print(f"API error: {e}") + return None - results = {'successful': [], 'failed': []} - - for model in models: - for benchmark in benchmarks: - print(f"Creating evaluation: {model} on {benchmark}") - - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - results['successful'].append({ - 'model': model, - 'benchmark': benchmark, - 'evaluation_id': evaluation.id - }) - print(f" Created: {evaluation.id}") - else: - results['failed'].append({ - 'model': model, - 'benchmark': benchmark, - 'error': 'No evaluation returned' - }) - - except atlas.RateLimitError: - print("Rate limited, waiting 60 seconds...") - time.sleep(60) - - except atlas.APIError as e: - print(f" Failed: {e}") - results['failed'].append({ - 'model': model, - 'benchmark': benchmark, - 'error': str(e) - }) - - time.sleep(2) - - return results - -# Usage -models = ["gpt-4", "claude-3-opus"] -benchmarks = ["mmlu", "hellaswag"] - -batch_results = run_evaluation_batch(models, benchmarks) -print(f" Successful: {len(batch_results['successful'])}") -print(f" Failed: {len(batch_results['failed'])}") +evaluation = safe_create_evaluation() ``` -## Error Handling Patterns - -### Robust Error Handling +### Retry Logic ```python import time from atlas import Atlas import atlas -def create_evaluation_with_retries(model, benchmark, max_retries=3): - """Create evaluation with automatic retries""" +def create_evaluation_with_retry(max_retries=3): client = Atlas() - + for attempt in range(max_retries): try: + models = client.models.get() + benchmarks = client.benchmarks.get() + evaluation = client.evaluations.create( - model=model, - benchmark=benchmark + model=models[0], + benchmark=benchmarks[0] ) - - if evaluation: - print(f" Success on attempt {attempt + 1}") - return evaluation - + + print(f"Success on attempt {attempt + 1}") + return evaluation + except atlas.RateLimitError as e: - print(f"Rate limited on attempt {attempt + 1}") if attempt < max_retries - 1: - # Check if server provided retry-after header - retry_after = getattr(e.response, 'headers', {}).get('retry-after', 60) - wait_time = int(retry_after) - print(f"Waiting {wait_time} seconds...") + wait_time = e.retry_after or (2 ** attempt) + print(f"Rate limited, waiting {wait_time}s...") time.sleep(wait_time) else: - raise - - except atlas.NotFoundError: - print(f" Model '{model}' or benchmark '{benchmark}' not found") - return None - - except atlas.AuthenticationError: - print(" Authentication failed - check your API key") - raise - + print("Max retries exceeded") + return None + except atlas.APIError as e: - print(f" API error on attempt {attempt + 1}: {e}") if attempt < max_retries - 1: + print(f"Attempt {attempt + 1} failed: {e}") time.sleep(2 ** attempt) # Exponential backoff else: - raise - + print(f"Failed after {max_retries} attempts: {e}") + return None + return None -# Usage -evaluation = create_evaluation_with_retries("gpt-4", "mmlu") +evaluation = create_evaluation_with_retry() ``` -## Result Processing +## Timeouts -### Processing Large Result Sets +### Custom Timeouts ```python from atlas import Atlas -import json -from typing import Dict, List -def analyze_evaluation_results(evaluation_id: str) -> Dict: - """Analyze results from an evaluation""" - client = Atlas() - - try: - results = client.results.get(evaluation_id=evaluation_id) - - if not results: - return {"error": "No results found"} - - # Basic analysis - analysis = { - "total_results": len(results), - "subsets": {}, - "overall_accuracy": 0, - "avg_duration": 0 - } - - total_score = 0 - total_duration = 0 - - for result in results: - # Track by subset - if result.subset not in analysis["subsets"]: - analysis["subsets"][result.subset] = { - "count": 0, - "total_score": 0, - "accuracy": 0 - } - - analysis["subsets"][result.subset]["count"] += 1 - analysis["subsets"][result.subset]["total_score"] += result.score - - total_score += result.score - total_duration += result.duration.total_seconds() - - # Calculate averages - analysis["overall_accuracy"] = total_score / len(results) - analysis["avg_duration"] = total_duration / len(results) +# Different timeout strategies +quick_client = Atlas(timeout=30.0) # 30 seconds for testing +normal_client = Atlas(timeout=300.0) # 5 minutes for normal use +patient_client = Atlas(timeout=1800.0) # 30 minutes for long evaluations - # Calculate subset accuracies - for subset_data in analysis["subsets"].values(): - subset_data["accuracy"] = subset_data["total_score"] / subset_data["count"] - - return analysis - - except atlas.APIError as e: - return {"error": str(e)} - -# Usage -analysis = analyze_evaluation_results("eval_123") -if "error" not in analysis: - print(f" Analysis Results:") - print(f" Total results: {analysis['total_results']}") - print(f" Overall accuracy: {analysis['overall_accuracy']:.2%}") - print(f" Average duration: {analysis['avg_duration']:.2f}s") - - print(f" By subset:") - for subset, data in analysis['subsets'].items(): - print(f" {subset}: {data['accuracy']:.2%} ({data['count']} results)") +# Use appropriate client for your use case +evaluation = patient_client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] +) ``` -## Production Timeouts - -### Different Timeout Strategies +### Per-Request Timeouts ```python from atlas import Atlas -# Different timeout configurations for different use cases - -# Development: Fail fast -dev_client = Atlas(timeout=30.0) # 30 seconds - -# Production: More patient -prod_client = Atlas(timeout=600.0) # 10 minutes - -# Long-running batch jobs: Very patient -batch_client = Atlas(timeout=1800.0) # 30 minutes - -def adaptive_timeout_client(operation_type="default"): - """Get client with timeout appropriate for operation""" - timeouts = { - "quick": 30.0, # For testing connectivity - "default": 300.0, # For normal operations - "batch": 1800.0, # For batch processing - "patient": 3600.0 # For very long evaluations - } +client = Atlas() - timeout = timeouts.get(operation_type, timeouts["default"]) - return Atlas(timeout=timeout) +# Override timeout for specific operations +models = client.models.get() +benchmarks = client.benchmarks.get() -# Usage -quick_client = adaptive_timeout_client("quick") -batch_client = adaptive_timeout_client("batch") +# Long evaluation with extended timeout +evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0], + timeout=3600.0 # 1 hour timeout for this specific request +) ``` -## Logging and Monitoring +## Batch Processing -### Simple Logging Setup +### Process Multiple Evaluations ```python -import logging -import time from atlas import Atlas -import atlas -# Set up logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) -logger = logging.getLogger('atlas-client') - -def create_evaluation_with_logging(model, benchmark): - """Create evaluation with comprehensive logging""" +def run_evaluation_batch(): client = Atlas() + + models = client.models.get() + benchmarks = client.benchmarks.get() + + results = [] + + # Create evaluations + for model in models[:2]: + for benchmark in benchmarks[:2]: + try: + evaluation = client.evaluations.create( + model=model, + benchmark=benchmark + ) + + print(f"Created: {model.id} + {benchmark.id} = {evaluation.id}") + results.append({ + 'model': model.id, + 'benchmark': benchmark.id, + 'evaluation_id': evaluation.id, + 'status': 'created' + }) + + # Small delay to avoid rate limits + time.sleep(1) + + except Exception as e: + print(f"Failed: {model.id} + {benchmark.id} - {e}") + results.append({ + 'model': model.id, + 'benchmark': benchmark.id, + 'error': str(e), + 'status': 'failed' + }) + + return results - logger.info(f"Creating evaluation: {model} on {benchmark}") - start_time = time.time() +batch_results = run_evaluation_batch() +``` - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) +## Pagination Best Practices - duration = time.time() - start_time +### Memory-Efficient Pagination - if evaluation: - logger.info( - f"Evaluation created successfully: {evaluation.id} " - f"(duration: {duration:.2f}s)" - ) - return evaluation - else: - logger.warning( - f"No evaluation returned for {model}+{benchmark} " - f"(duration: {duration:.2f}s)" - ) - return None +```python +from atlas import Atlas - except atlas.APIError as e: - duration = time.time() - start_time - logger.error( - f"Failed to create evaluation {model}+{benchmark}: {e} " - f"(duration: {duration:.2f}s)" +def process_large_results(evaluation_id): + """Process large result sets without loading everything into memory""" + client = Atlas() + + page = 1 + total_processed = 0 + total_correct = 0 + + while True: + # Get one page at a time + results_data = client.results.get_by_id( + evaluation_id=evaluation_id, + page=page, + page_size=100 ) - raise + + if not results_data or not results_data.results: + break + + # Process this page + page_correct = sum(1 for r in results_data.results if r.score > 0.5) + total_correct += page_correct + total_processed += len(results_data.results) + + # Show progress + accuracy = total_correct / total_processed + print(f"Page {page}: {accuracy:.1%} accuracy ({total_processed:,} processed)") + + if page >= results_data.pagination.total_pages: + break + + page += 1 + + final_accuracy = total_correct / total_processed if total_processed > 0 else 0 + print(f"Final: {final_accuracy:.1%} accuracy ({total_processed:,} total)") + + return final_accuracy -# Usage -evaluation = create_evaluation_with_logging("gpt-4", "mmlu") +accuracy = process_large_results("your_evaluation_id") ``` ## Health Checks @@ -429,146 +309,68 @@ from atlas import Atlas import atlas def check_atlas_health(): - """Simple health check for Atlas service""" + """Check if Atlas API is reachable""" try: - client = Atlas(timeout=10.0) # Short timeout for health check - - # Try to create a test evaluation (will fail but tests connectivity) - try: - client.evaluations.create( - model="__health_check__", - benchmark="__health_check__" - ) - except atlas.NotFoundError: - # Expected - health check resources don't exist - return {"status": "healthy", "message": "API is reachable"} - except atlas.BadRequestError: - # Also expected - invalid parameters - return {"status": "healthy", "message": "API is reachable"} - - except atlas.AuthenticationError: + client = Atlas(timeout=10.0) + + # Try to get models (quick operation) + models = client.models.get() + return { - "status": "unhealthy", - "error": "Authentication failed - check API key" + 'status': 'healthy', + 'models_count': len(models) } + + except atlas.AuthenticationError: + return {'status': 'unhealthy', 'error': 'authentication_failed'} except atlas.APIConnectionError: - return { - "status": "unhealthy", - "error": "Cannot connect to Atlas API" - } + return {'status': 'unhealthy', 'error': 'connection_failed'} except atlas.APITimeoutError: - return { - "status": "unhealthy", - "error": "Health check timed out" - } + return {'status': 'unhealthy', 'error': 'timeout'} except Exception as e: - return { - "status": "unhealthy", - "error": f"Unexpected error: {e}" - } + return {'status': 'unhealthy', 'error': str(e)} -# Usage health = check_atlas_health() -if health["status"] == "healthy": - print(" Atlas service is healthy") +if health['status'] == 'healthy': + print(f"Atlas is healthy ({health['models_count']} models available)") else: - print(f" Atlas service is unhealthy: {health['error']}") + print(f"Atlas is unhealthy: {health['error']}") ``` -## Integration Patterns +## Monitoring and Logging -### Using with Flask/FastAPI +### Basic Logging ```python -from flask import Flask, jsonify, request +import logging from atlas import Atlas -import atlas - -app = Flask(__name__) - -# Initialize Atlas client once -atlas_client = Atlas() -@app.route('/health') -def health_check(): - """Health check endpoint""" - health = check_atlas_health() # From example above - status_code = 200 if health["status"] == "healthy" else 503 - return jsonify(health), status_code +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) -@app.route('/evaluations', methods=['POST']) -def create_evaluation(): - """Create evaluation endpoint""" +def create_evaluation_with_logging(): + client = Atlas() + + logger.info("Starting evaluation creation") + try: - data = request.get_json() - model = data.get('model') - benchmark = data.get('benchmark') - - if not model or not benchmark: - return jsonify({ - "error": "Missing required fields: model, benchmark" - }), 400 - - evaluation = atlas_client.evaluations.create( - model=model, - benchmark=benchmark + models = client.models.get() + benchmarks = client.benchmarks.get() + + logger.info(f"Found {len(models)} models, {len(benchmarks)} benchmarks") + + evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] ) + + logger.info(f"Created evaluation: {evaluation.id}") + return evaluation + + except Exception as e: + logger.error(f"Failed to create evaluation: {e}") + return None - if evaluation: - return jsonify({ - "success": True, - "evaluation_id": evaluation.id, - "status": evaluation.status - }) - else: - return jsonify({ - "success": False, - "error": "Failed to create evaluation" - }), 500 - - except atlas.NotFoundError: - return jsonify({ - "success": False, - "error": "Model or benchmark not found" - }), 404 - - except atlas.APIError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 500 - -@app.route('/evaluations//results') -def get_results(evaluation_id): - """Get evaluation results endpoint""" - try: - results = atlas_client.results.get(evaluation_id=evaluation_id) - - if results: - return jsonify({ - "success": True, - "result_count": len(results), - "results": [ - { - "subset": r.subset, - "score": r.score, - "duration_seconds": r.duration.total_seconds() - } - for r in results - ] - }) - else: - return jsonify({ - "success": False, - "error": "No results found" - }), 404 - - except atlas.APIError as e: - return jsonify({ - "success": False, - "error": str(e) - }), 500 - -if __name__ == '__main__': - app.run(debug=True) +evaluation = create_evaluation_with_logging() ``` diff --git a/docs/examples/creating-evaluations.md b/docs/examples/creating-evaluations.md index 78e2900..e77c9fb 100644 --- a/docs/examples/creating-evaluations.md +++ b/docs/examples/creating-evaluations.md @@ -1,795 +1,152 @@ # Creating Evaluations -This guide provides practical examples for creating evaluations with the Atlas Python SDK. +Simple examples for creating AI model evaluations with the Atlas Python SDK. -## Basic Evaluation Creation +## Quick Start -### Simple Evaluation - -The most straightforward way to create an evaluation: +### Basic Evaluation ```python from atlas import Atlas -# Initialize client +# Initialize client (reads LAYERLENS_ATLAS_API_KEY from environment) client = Atlas() -# Create evaluation -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" -) - -if evaluation: - print(f"āœ… Evaluation created: {evaluation.id}") - print(f" Model: {evaluation.model_name}") - print(f" Benchmark: {evaluation.dataset_name}") - print(f" Status: {evaluation.status}") -else: - print("āŒ Failed to create evaluation") -``` - -### With Explicit Configuration +# Get available models and benchmarks +models = client.models.get() +benchmarks = client.benchmarks.get() -Using explicit client configuration instead of environment variables: - -```python -from atlas import Atlas - -# Explicit configuration -client = Atlas(api_key="your_api_key_here") +print(f"Available: {len(models)} models, {len(benchmarks)} benchmarks") +# Create evaluation with first available model and benchmark evaluation = client.evaluations.create( - model="claude-3-opus", - benchmark="hellaswag" + model=models[0], + benchmark=benchmarks[0] ) -if evaluation: - print(f"Evaluation ID: {evaluation.id}") - print(f"Submitted at: {evaluation.submitted_at}") +print(f"Created evaluation: {evaluation.id}") +print(f"Status: {evaluation.status}") ``` -## Batch Evaluation Creation - -### Multiple Models on Same Benchmark - -Compare multiple models against the same benchmark: +### Choose Specific Model and Benchmark ```python from atlas import Atlas -import time - -def compare_models_on_benchmark(models: list, benchmark: str): - """Create evaluations for multiple models on the same benchmark""" - client = Atlas() - evaluations = [] - - print(f"šŸ”„ Creating evaluations for {len(models)} models on {benchmark}") - - for model in models: - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - evaluations.append({ - "model": model, - "evaluation_id": evaluation.id, - "model_name": evaluation.model_name, - "status": evaluation.status - }) - print(f"āœ… {model}: {evaluation.id}") - else: - print(f"āŒ Failed to create evaluation for {model}") - - except Exception as e: - print(f"āŒ Error creating evaluation for {model}: {e}") - - # Brief pause between requests to avoid rate limits - time.sleep(0.5) - - return evaluations - -# Usage -models_to_compare = [ - "gpt-4", - "gpt-3.5-turbo", - "claude-3-opus", - "claude-3-sonnet", - "llama-2-70b" -] - -evaluations = compare_models_on_benchmark(models_to_compare, "mmlu") - -# Print summary -print(f"\nšŸ“Š Created {len(evaluations)} evaluations:") -for eval_info in evaluations: - print(f" {eval_info['model_name']}: {eval_info['evaluation_id']}") -``` -### Single Model on Multiple Benchmarks - -Evaluate one model across multiple benchmarks: - -```python -from atlas import Atlas -import time - -def evaluate_model_on_benchmarks(model: str, benchmarks: list): - """Evaluate a single model across multiple benchmarks""" - client = Atlas() - evaluations = [] - - print(f"šŸ”„ Evaluating {model} on {len(benchmarks)} benchmarks") - - for benchmark in benchmarks: - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - evaluations.append({ - "benchmark": benchmark, - "evaluation_id": evaluation.id, - "dataset_name": evaluation.dataset_name, - "status": evaluation.status - }) - print(f"āœ… {benchmark}: {evaluation.id}") - else: - print(f"āŒ Failed to create evaluation for {benchmark}") - - except Exception as e: - print(f"āŒ Error evaluating on {benchmark}: {e}") - - time.sleep(0.5) - - return evaluations - -# Usage -benchmarks_to_test = [ - "mmlu", - "hellaswag", - "arc-challenge", - "truthfulqa", - "gsm8k" -] - -evaluations = evaluate_model_on_benchmarks("gpt-4", benchmarks_to_test) - -print(f"\nšŸ“Š Created {len(evaluations)} evaluations for GPT-4:") -for eval_info in evaluations: - print(f" {eval_info['dataset_name']}: {eval_info['evaluation_id']}") -``` - -### Full Matrix Evaluation - -Create evaluations for all model-benchmark combinations: - -```python -from atlas import Atlas -import time -import itertools - -def create_evaluation_matrix(models: list, benchmarks: list, delay: float = 1.0): - """Create evaluations for all model-benchmark combinations""" - client = Atlas() - results = {} - total_combinations = len(models) * len(benchmarks) - - print(f"šŸ”„ Creating {total_combinations} evaluations...") - - for i, (model, benchmark) in enumerate(itertools.product(models, benchmarks), 1): - print(f"\n[{i}/{total_combinations}] {model} + {benchmark}") - - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - if model not in results: - results[model] = {} - - results[model][benchmark] = { - "evaluation_id": evaluation.id, - "model_name": evaluation.model_name, - "dataset_name": evaluation.dataset_name, - "status": evaluation.status, - "success": True - } - print(f"āœ… Success: {evaluation.id}") - else: - print(f"āŒ Failed: No evaluation created") - - except Exception as e: - print(f"āŒ Error: {e}") - if model not in results: - results[model] = {} - results[model][benchmark] = { - "error": str(e), - "success": False - } - - # Rate limiting - if i < total_combinations: - time.sleep(delay) - - return results - -# Usage -test_models = ["gpt-4", "claude-3-opus", "llama-2-70b"] -test_benchmarks = ["mmlu", "hellaswag", "arc-challenge"] - -matrix_results = create_evaluation_matrix(test_models, test_benchmarks, delay=2.0) - -# Print summary table -print(f"\nšŸ“Š Evaluation Matrix Results:") -print("Model".ljust(15), end="") -for benchmark in test_benchmarks: - print(benchmark.ljust(15), end="") -print() - -for model in test_models: - print(model.ljust(15), end="") - for benchmark in test_benchmarks: - if model in matrix_results and benchmark in matrix_results[model]: - result = matrix_results[model][benchmark] - status = "āœ…" if result["success"] else "āŒ" - print(status.ljust(15), end="") - else: - print("ā“".ljust(15), end="") - print() -``` - -## Error Handling and Resilience +client = Atlas() -### Robust Evaluation Creation with Retries +# Get all available options +models = client.models.get() +benchmarks = client.benchmarks.get() -```python -import atlas -from atlas import Atlas -import time -import random - -def create_evaluation_with_retry( - model: str, - benchmark: str, - max_retries: int = 3, - base_delay: float = 1.0 -): - """Create evaluation with exponential backoff retry logic""" - client = Atlas() - - for attempt in range(max_retries): - try: - print(f"šŸ”„ Attempt {attempt + 1}/{max_retries}: Creating evaluation...") - - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark, - timeout=120.0 # 2-minute timeout - ) - - if evaluation: - print(f"āœ… Success on attempt {attempt + 1}: {evaluation.id}") - return evaluation - else: - print(f"āŒ Evaluation creation returned None on attempt {attempt + 1}") - - except atlas.RateLimitError as e: - retry_after = e.response.headers.get('retry-after', base_delay * (2 ** attempt)) - print(f"ā³ Rate limited, waiting {retry_after}s...") - time.sleep(float(retry_after)) - continue - - except atlas.InternalServerError: - if attempt < max_retries - 1: - delay = base_delay * (2 ** attempt) + random.uniform(0, 1) - print(f"šŸ”„ Server error, retrying in {delay:.1f}s...") - time.sleep(delay) - continue - else: - print("āŒ Server error - max retries exceeded") - break - - except atlas.APIConnectionError: - if attempt < max_retries - 1: - delay = base_delay * (2 ** attempt) - print(f"šŸ”„ Connection error, retrying in {delay:.1f}s...") - time.sleep(delay) - continue - else: - print("āŒ Connection failed - max retries exceeded") - break - - except atlas.AuthenticationError: - print("āŒ Authentication failed - check your API key") - break - - except atlas.NotFoundError: - print(f"āŒ Model '{model}' or benchmark '{benchmark}' not found") - break - - except atlas.PermissionDeniedError: - print("āŒ Permission denied - check your access rights") - break - - except atlas.APIError as e: - print(f"āŒ API error: {e}") - break - - return None - -# Usage -evaluation = create_evaluation_with_retry( - model="gpt-4", - benchmark="mmlu", - max_retries=3 -) +# Find specific model and benchmark +gpt4_model = next((m for m in models if "gpt-4" in m.id), None) +mmlu_benchmark = next((b for b in benchmarks if "mmlu" in b.id), None) -if evaluation: - print(f"Final result: {evaluation.id}") +if gpt4_model and mmlu_benchmark: + evaluation = client.evaluations.create( + model=gpt4_model, + benchmark=mmlu_benchmark + ) + print(f"Created: {evaluation.id}") else: - print("Failed to create evaluation after all attempts") + print("Model or benchmark not found") + print(f"Available models: {[m.id for m in models[:5]]}...") + print(f"Available benchmarks: {[b.id for b in benchmarks[:5]]}...") ``` -### Validation Before Creation +## Async Version ```python -import atlas -from atlas import Atlas +import asyncio +from atlas import AsyncAtlas + +async def create_evaluation(): + client = AsyncAtlas() + + models = await client.models.get() + benchmarks = await client.benchmarks.get() + + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] + ) + + print(f"Created evaluation: {evaluation.id}") + return evaluation -def validate_and_create_evaluation(model: str, benchmark: str): - """Validate model and benchmark before creating evaluation""" - client = Atlas() - - # Pre-validation checks - if not model or not model.strip(): - print("āŒ Model cannot be empty") - return None - - if not benchmark or not benchmark.strip(): - print("āŒ Benchmark cannot be empty") - return None - - print(f"šŸ” Validating {model} + {benchmark}...") - - try: - # Attempt to create the evaluation - evaluation = client.evaluations.create( - model=model.strip(), - benchmark=benchmark.strip() - ) - - if evaluation: - print(f"āœ… Validation successful!") - print(f" Evaluation ID: {evaluation.id}") - print(f" Model: {evaluation.model_name} ({evaluation.model_company})") - print(f" Benchmark: {evaluation.dataset_name}") - print(f" Status: {evaluation.status}") - return evaluation - else: - print("āŒ Validation failed: No evaluation returned") - return None - - except atlas.NotFoundError: - print(f"āŒ Validation failed: Model '{model}' or benchmark '{benchmark}' not found") - print("šŸ’” Suggestions:") - print(" ā€¢ Check spelling of model and benchmark IDs") - print(" ā€¢ Verify available options in Atlas dashboard") - print(" ā€¢ Ensure your organization has access to these resources") - return None - - except atlas.AuthenticationError: - print("āŒ Authentication failed") - print("šŸ’” Check your API key configuration") - return None - - except atlas.PermissionDeniedError: - print("āŒ Permission denied") - print("šŸ’” Contact your administrator for access") - return None - - except atlas.APIError as e: - print(f"āŒ Validation failed: {e}") - return None - -# Usage with validation -test_combinations = [ - ("gpt-4", "mmlu"), - ("claude-3-opus", "hellaswag"), - ("nonexistent-model", "mmlu"), # This should fail - ("gpt-4", "nonexistent-benchmark"), # This should fail -] - -for model, benchmark in test_combinations: - print(f"\n{'='*50}") - evaluation = validate_and_create_evaluation(model, benchmark) - - if evaluation: - print(f"Ready to monitor evaluation: {evaluation.id}") +# Run it +asyncio.run(create_evaluation()) ``` -## Custom Timeout Configurations - -### Different Timeouts for Different Operations +## Wait for Completion ```python from atlas import Atlas -import httpx - -def create_evaluations_with_custom_timeouts(): - """Demonstrate different timeout configurations""" - - # Quick timeout for testing connectivity - quick_client = Atlas(timeout=30.0) # 30 seconds - - # Standard timeout for regular evaluations - standard_client = Atlas(timeout=300.0) # 5 minutes - - # Long timeout for complex evaluations - patient_client = Atlas( - timeout=httpx.Timeout( - connect=10.0, # 10s to connect - read=1800.0, # 30min to read response - write=60.0, # 1min to send request - pool=30.0 # 30s for connection pool - ) - ) - - # Test connectivity with quick client - print("šŸ” Testing connectivity...") - try: - test_eval = quick_client.evaluations.create( - model="gpt-3.5-turbo", # Faster model for testing - benchmark="arc-easy" # Smaller benchmark for testing - ) - print("āœ… Connectivity test passed") - except atlas.APITimeoutError: - print("āŒ Quick connectivity test failed - network issues?") - return - except atlas.APIError as e: - print(f"āŒ API error during connectivity test: {e}") - return - - # Create standard evaluation - print("\nšŸ”„ Creating standard evaluation...") - try: - standard_eval = standard_client.evaluations.create( - model="gpt-4", - benchmark="mmlu" - ) - if standard_eval: - print(f"āœ… Standard evaluation created: {standard_eval.id}") - except atlas.APITimeoutError: - print("āŒ Standard evaluation timed out") - - # Create complex evaluation with patient timeout - print("\nšŸ”„ Creating complex evaluation...") - try: - complex_eval = patient_client.evaluations.create( - model="gpt-4", - benchmark="math" # Complex benchmark - ) - if complex_eval: - print(f"āœ… Complex evaluation created: {complex_eval.id}") - except atlas.APITimeoutError: - print("āŒ Complex evaluation timed out even with extended timeout") - -# Run the example -create_evaluations_with_custom_timeouts() -``` -### Per-Request Timeout Override +client = Atlas() +models = client.models.get() +benchmarks = client.benchmarks.get() -```python -from atlas import Atlas +# Create evaluation +evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) +print(f"Created: {evaluation.id}") + +# Wait for it to complete (this may take several minutes) +completed_evaluation = client.evaluations.wait_for_completion( + evaluation, + interval_seconds=30, # Check every 30 seconds + timeout=1800 # 30 minute timeout +) -def create_evaluation_with_override_timeout(): - """Override timeout for specific requests""" - client = Atlas(timeout=60.0) # Default 1-minute timeout - - evaluations = [] - - # Quick evaluation with short timeout - print("šŸ”„ Quick evaluation (30s timeout)...") - try: - quick_eval = client.with_options(timeout=30.0).evaluations.create( - model="gpt-3.5-turbo", - benchmark="arc-easy" - ) - if quick_eval: - evaluations.append(("Quick", quick_eval)) - print(f"āœ… Quick: {quick_eval.id}") - except atlas.APITimeoutError: - print("āŒ Quick evaluation timed out") - - # Standard evaluation (uses default timeout) - print("\nšŸ”„ Standard evaluation (default 60s timeout)...") - try: - standard_eval = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" - ) - if standard_eval: - evaluations.append(("Standard", standard_eval)) - print(f"āœ… Standard: {standard_eval.id}") - except atlas.APITimeoutError: - print("āŒ Standard evaluation timed out") - - # Long evaluation with extended timeout - print("\nšŸ”„ Long evaluation (5min timeout)...") - try: - long_eval = client.with_options(timeout=300.0).evaluations.create( - model="gpt-4", - benchmark="math" - ) - if long_eval: - evaluations.append(("Long", long_eval)) - print(f"āœ… Long: {long_eval.id}") - except atlas.APITimeoutError: - print("āŒ Long evaluation timed out") - - return evaluations - -evaluations = create_evaluation_with_override_timeout() -print(f"\nšŸ“Š Created {len(evaluations)} evaluations total") +if completed_evaluation.is_success: + print("Evaluation completed successfully!") +else: + print(f"Evaluation failed: {completed_evaluation.status}") ``` -## Monitoring and Logging - -### Evaluation Creation with Logging +## Error Handling ```python -import logging -from datetime import datetime from atlas import Atlas import atlas -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('atlas_evaluations.log'), - logging.StreamHandler() - ] -) -logger = logging.getLogger(__name__) - -def create_evaluation_with_logging(model: str, benchmark: str, context: dict = None): - """Create evaluation with comprehensive logging""" - client = Atlas() - context = context or {} - - logger.info(f"Starting evaluation creation: {model} + {benchmark}") - logger.info(f"Context: {context}") - - start_time = datetime.now() - - try: - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - end_time = datetime.now() - duration = (end_time - start_time).total_seconds() - - if evaluation: - logger.info(f"āœ… Evaluation created successfully in {duration:.2f}s") - logger.info(f" ID: {evaluation.id}") - logger.info(f" Model: {evaluation.model_name} ({evaluation.model_company})") - logger.info(f" Benchmark: {evaluation.dataset_name}") - logger.info(f" Status: {evaluation.status}") - logger.info(f" Submitted at: {evaluation.submitted_at}") - - return { - "success": True, - "evaluation": evaluation, - "duration": duration, - "timestamp": start_time.isoformat() - } - else: - logger.error(f"āŒ Evaluation creation failed - returned None") - return { - "success": False, - "error": "No evaluation returned", - "duration": duration, - "timestamp": start_time.isoformat() - } - - except atlas.RateLimitError as e: - logger.warning(f"ā³ Rate limited - request ID: {getattr(e, 'request_id', 'N/A')}") - return {"success": False, "error": "rate_limited", "retry_after": e.response.headers.get('retry-after')} - - except atlas.AuthenticationError: - logger.error("āŒ Authentication failed - check API key") - return {"success": False, "error": "authentication_failed"} - - except atlas.NotFoundError: - logger.error(f"āŒ Model '{model}' or benchmark '{benchmark}' not found") - return {"success": False, "error": "not_found", "model": model, "benchmark": benchmark} - - except atlas.APIError as e: - logger.error(f"āŒ API error: {e}") - return {"success": False, "error": str(e), "error_type": type(e).__name__} - - except Exception as e: - logger.error(f"āŒ Unexpected error: {e}") - return {"success": False, "error": f"unexpected: {e}"} - -# Usage -evaluation_configs = [ - {"model": "gpt-4", "benchmark": "mmlu", "context": {"purpose": "baseline_test"}}, - {"model": "claude-3-opus", "benchmark": "hellaswag", "context": {"purpose": "reasoning_comparison"}}, - {"model": "llama-2-70b", "benchmark": "gsm8k", "context": {"purpose": "math_evaluation"}}, -] - -results = [] -for config in evaluation_configs: - result = create_evaluation_with_logging(**config) - results.append(result) - - if not result["success"]: - logger.error(f"Failed to create evaluation: {config}") - -# Summary -successful = [r for r in results if r["success"]] -failed = [r for r in results if not r["success"]] - -logger.info(f"šŸ“Š Summary: {len(successful)} successful, {len(failed)} failed") -for result in successful: - logger.info(f" āœ… {result['evaluation'].id} ({result['duration']:.2f}s)") -for result in failed: - logger.info(f" āŒ {result.get('error', 'unknown_error')}") -``` +client = Atlas() -## Advanced Patterns +try: + models = client.models.get() + benchmarks = client.benchmarks.get() + + evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] + ) + print(f"Success: {evaluation.id}") + +except atlas.AuthenticationError: + print("Check your API key") +except atlas.NotFoundError: + print("Model or benchmark not found") +except atlas.APIError as e: + print(f"API error: {e}") +``` -### Evaluation Factory Pattern +## Multiple Evaluations ```python from atlas import Atlas -from abc import ABC, abstractmethod -from typing import List, Dict, Any -import atlas -class EvaluationStrategy(ABC): - """Abstract base class for evaluation strategies""" - - @abstractmethod - def get_model_benchmark_pairs(self) -> List[tuple]: - pass - - @abstractmethod - def get_description(self) -> str: - pass - -class GeneralIntelligenceStrategy(EvaluationStrategy): - """Strategy for general intelligence assessment""" - - def get_model_benchmark_pairs(self) -> List[tuple]: - models = ["gpt-4", "claude-3-opus", "llama-2-70b"] - benchmarks = ["mmlu", "arc-challenge", "hellaswag"] - return [(m, b) for m in models for b in benchmarks] - - def get_description(self) -> str: - return "General intelligence assessment across major benchmarks" - -class CodeGenerationStrategy(EvaluationStrategy): - """Strategy for code generation assessment""" - - def get_model_benchmark_pairs(self) -> List[tuple]: - models = ["gpt-4", "code-llama-34b", "claude-3-sonnet"] - benchmarks = ["humaneval", "mbpp"] - return [(m, b) for m in models for b in benchmarks] - - def get_description(self) -> str: - return "Code generation capability assessment" - -class MathReasoningStrategy(EvaluationStrategy): - """Strategy for mathematical reasoning assessment""" - - def get_model_benchmark_pairs(self) -> List[tuple]: - models = ["gpt-4", "claude-3-opus", "minerva-62b"] - benchmarks = ["gsm8k", "math"] - return [(m, b) for m in models for b in benchmarks] - - def get_description(self) -> str: - return "Mathematical reasoning and problem-solving assessment" - -class EvaluationFactory: - """Factory for creating evaluations based on strategies""" - - def __init__(self): - self.client = Atlas() - - def execute_strategy(self, strategy: EvaluationStrategy) -> Dict[str, Any]: - """Execute an evaluation strategy""" - pairs = strategy.get_model_benchmark_pairs() - description = strategy.get_description() - - print(f"šŸ”„ Executing strategy: {description}") - print(f"šŸ“Š Creating {len(pairs)} evaluations...") - - results = { - "strategy": description, - "evaluations": [], - "errors": [], - "summary": {"total": len(pairs), "successful": 0, "failed": 0} - } - - for model, benchmark in pairs: - try: - evaluation = self.client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - results["evaluations"].append({ - "model": model, - "benchmark": benchmark, - "evaluation_id": evaluation.id, - "model_name": evaluation.model_name, - "dataset_name": evaluation.dataset_name, - "status": evaluation.status - }) - results["summary"]["successful"] += 1 - print(f"āœ… {model} + {benchmark}: {evaluation.id}") - else: - results["errors"].append({ - "model": model, - "benchmark": benchmark, - "error": "No evaluation returned" - }) - results["summary"]["failed"] += 1 - print(f"āŒ {model} + {benchmark}: Failed") - - except atlas.APIError as e: - results["errors"].append({ - "model": model, - "benchmark": benchmark, - "error": str(e), - "error_type": type(e).__name__ - }) - results["summary"]["failed"] += 1 - print(f"āŒ {model} + {benchmark}: {e}") - - return results - -# Usage -factory = EvaluationFactory() - -# Run different strategies -strategies = [ - GeneralIntelligenceStrategy(), - CodeGenerationStrategy(), - MathReasoningStrategy() -] - -all_results = [] -for strategy in strategies: - result = factory.execute_strategy(strategy) - all_results.append(result) - - print(f"\nšŸ“ˆ Strategy Results: {result['strategy']}") - print(f" Successful: {result['summary']['successful']}") - print(f" Failed: {result['summary']['failed']}") - print() - -# Overall summary -total_evaluations = sum(r["summary"]["successful"] for r in all_results) -total_errors = sum(r["summary"]["failed"] for r in all_results) - -print(f"šŸŽÆ Overall Summary:") -print(f" Total evaluations created: {total_evaluations}") -print(f" Total errors: {total_errors}") -print(f" Success rate: {total_evaluations/(total_evaluations+total_errors)*100:.1f}%") +client = Atlas() +models = client.models.get() +benchmarks = client.benchmarks.get() + +# Create multiple evaluations +evaluations = [] +for model in models[:3]: # First 3 models + for benchmark in benchmarks[:2]: # First 2 benchmarks + evaluation = client.evaluations.create(model=model, benchmark=benchmark) + evaluations.append(evaluation) + print(f"Created: {model.id} + {benchmark.id} = {evaluation.id}") + +print(f"Created {len(evaluations)} evaluations total") ``` diff --git a/docs/examples/retrieving-results.md b/docs/examples/retrieving-results.md index 19c6b68..06f9b64 100644 --- a/docs/examples/retrieving-results.md +++ b/docs/examples/retrieving-results.md @@ -1,322 +1,93 @@ # Retrieving Results -This guide provides practical examples for retrieving and analyzing evaluation results with the Atlas Python SDK. +Simple examples for getting evaluation results with the Atlas Python SDK. ## Basic Result Retrieval -### Simple Result Fetching +### Get Results (First Page) ```python from atlas import Atlas -# Initialize client client = Atlas() # Get results for a specific evaluation -evaluation_id = "eval_12345" # Replace with your evaluation ID -results_data = client.results.get(evaluation_id=evaluation_id) +evaluation_id = "your_evaluation_id" +results = client.results.get_by_id(evaluation_id=evaluation_id) -if results_data: - print(f"Evaluation: {results_data.evaluation_id}") - print(f"Retrieved {len(results_data.results)} results (page 1)") - print(f"Total available: {results_data.pagination.total_count}") - print(f"Total pages: {results_data.pagination.total_pages}") +if results: + print(f"Results for evaluation: {results.evaluation_id}") + print(f"Total results: {results.pagination.total_count:,}") + print(f"Showing page 1 of {results.pagination.total_pages}") + print(f"Results on this page: {len(results.results)}") # Show first few results - for i, result in enumerate(results_data.results[:3]): + for i, result in enumerate(results.results[:3]): print(f"\nResult {i+1}:") + print(f" Score: {result.score}") print(f" Subset: {result.subset}") print(f" Prompt: {result.prompt[:100]}...") - print(f" Model Response: {result.result[:100]}...") - print(f" Expected: {result.truth}") - print(f" Score: {result.score}") - print(f" Duration: {result.duration}") + print(f" Response: {result.result[:50]}...") else: print("No results found") ``` -### Paginated Result Retrieval - -```python -from atlas import Atlas - -# Initialize client -client = Atlas() - -def get_paginated_results(evaluation_id: str, page_size: int = 50): - """Get results with pagination control""" - - # Get specific page - results_data = client.results.get( - evaluation_id=evaluation_id, - page=2, # Get second page - page_size=page_size - ) - - if results_data: - pagination = results_data.pagination - print(f"Pagination Info:") - print(f" Total results: {pagination.total_count}") - print(f" Page size: {pagination.page_size}") - print(f" Total pages: {pagination.total_pages}") - print(f" Current page results: {len(results_data.results)}") - - return results_data - else: - print("No results found") - return None - -# Usage -paginated_results = get_paginated_results("eval_12345", page_size=25) -``` +## Pagination - Get All Results -### Complete Evaluation Workflow +### Simple Approach ```python from atlas import Atlas -import time -def complete_evaluation_workflow(model: str, benchmark: str): - """Complete workflow: create evaluation and retrieve results""" - client = Atlas() - - # Step 1: Create evaluation - print(f"Creating evaluation: {model} + {benchmark}") - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - - if not evaluation: - print("Failed to create evaluation") - return None - - print(f"Evaluation created: {evaluation.id}") - print(f" Status: {evaluation.status}") - - # Step 2: Wait for completion (simplified polling) - # In production, use webhooks instead of polling - print("Waiting for evaluation to complete...") - - # Note: This is a simplified example. In practice, you'd: - # 1. Use webhooks for real-time updates - # 2. Store evaluation ID and check periodically - # 3. Handle various status states properly - - if evaluation.status == "completed": - print("Evaluation completed!") - - # Step 3: Retrieve results - results_data = client.results.get(evaluation_id=evaluation.id) - - if results_data: - results = results_data.results - print(f"Retrieved {len(results)} results from page 1") - print(f"Total results available: {results_data.pagination.total_count}") - - # Basic analysis for current page - correct_answers = sum(1 for r in results if r.score > 0.5) - accuracy = correct_answers / len(results) - avg_duration = sum(r.duration for r in results) / len(results) - - print(f"Quick Analysis (Page 1):") - print(f" Accuracy: {accuracy:.1%} ({correct_answers}/{len(results)})") - print(f" Average Duration: {avg_duration}") - - # Note about pagination - if results_data.pagination.total_pages > 1: - print(f"Note: This evaluation has {results_data.pagination.total_pages} pages total") - print(f" Use pagination to process all {results_data.pagination.total_count} results") - - return results_data - else: - print("No results available") - else: - print(f"Evaluation status: {evaluation.status}") - print(" Check back later for results") - - return None - -# Usage -results = complete_evaluation_workflow("gpt-4", "mmlu") -``` - -## Result Analysis Patterns - -### Performance Analysis - -```python -from atlas import Atlas -from collections import defaultdict, Counter -import statistics -from datetime import timedelta +client = Atlas() +evaluation_id = "your_evaluation_id" -def analyze_evaluation_performance(evaluation_id: str, use_all_pages: bool = True): - """Comprehensive performance analysis of evaluation results""" - client = Atlas() - - if use_all_pages: - # Get all results across all pages for complete analysis +# Get all results across all pages all_results = [] page = 1 - page_size = 100 while True: - results_data = client.results.get( + results_data = client.results.get_by_id( evaluation_id=evaluation_id, page=page, - page_size=page_size + page_size=100 # 100 results per page ) if not results_data or not results_data.results: break all_results.extend(results_data.results) - - # Use pagination info from the first page - if page == 1: - total_count = results_data.pagination.total_count - total_pages = results_data.pagination.total_pages - print(f"Loading {total_count} results from {total_pages} pages...") - - print(f" Loaded page {page}/{total_pages}") - + print(f"Loaded page {page}: {len(results_data.results)} results") + + # Check if we've reached the last page if page >= results_data.pagination.total_pages: break page += 1 - results = all_results - - if not results: - print(f"No results found for evaluation {evaluation_id}") - return None - - else: - # Analyze just the first page - results_data = client.results.get(evaluation_id=evaluation_id, page=1, page_size=100) - if not results_data: - print(f"No results found for evaluation {evaluation_id}") - return None - - results = results_data.results - print(f"Analyzing first page only ({len(results)} of {results_data.pagination.total_count} total results)") - - print(f"Performance Analysis for {evaluation_id}") - print(f"{'='*60}") - - # Overall statistics - total_cases = len(results) - correct_answers = sum(1 for r in results if r.score > 0.5) - total_score = sum(r.score for r in results) - - accuracy = correct_answers / total_cases - avg_score = total_score / total_cases - - print(f"\nšŸŽÆ Overall Performance:") - print(f" Total test cases: {total_cases:,}") - print(f" Correct answers: {correct_answers:,}") - print(f" Accuracy: {accuracy:.1%}") - print(f" Average score: {avg_score:.3f}") - - # Timing analysis - durations = [r.duration for r in results] - avg_duration = sum(durations, timedelta()) / len(durations) - min_duration = min(durations) - max_duration = max(durations) - median_duration = statistics.median(durations) - - print(f"\nā±ļø Timing Analysis:") - print(f" Average duration: {avg_duration}") - print(f" Median duration: {median_duration}") - print(f" Min duration: {min_duration}") - print(f" Max duration: {max_duration}") - - # Score distribution - score_ranges = { - "Perfect (1.0)": 0, - "High (0.8-0.99)": 0, - "Medium (0.5-0.79)": 0, - "Low (0.1-0.49)": 0, - "Zero (0.0)": 0 - } - - for result in results: - score = result.score - if score == 1.0: - score_ranges["Perfect (1.0)"] += 1 - elif 0.8 <= score < 1.0: - score_ranges["High (0.8-0.99)"] += 1 - elif 0.5 <= score < 0.8: - score_ranges["Medium (0.5-0.79)"] += 1 - elif 0.1 <= score < 0.5: - score_ranges["Low (0.1-0.49)"] += 1 - else: - score_ranges["Zero (0.0)"] += 1 - - print(f"\nScore Distribution:") - for range_name, count in score_ranges.items(): - percentage = count / total_cases * 100 - print(f" {range_name}: {count:,} ({percentage:.1f}%)") - - # Subset analysis - subset_stats = defaultdict(lambda: {"scores": [], "durations": []}) - - for result in results: - subset_stats[result.subset]["scores"].append(result.score) - subset_stats[result.subset]["durations"].append(result.duration) - - print(f"\nPerformance by Subset:") - print(f"{'Subset':<25} {'Cases':<8} {'Accuracy':<10} {'Avg Score':<10} {'Avg Duration':<12}") - print("-" * 75) - - for subset, data in sorted(subset_stats.items()): - case_count = len(data["scores"]) - subset_accuracy = sum(1 for s in data["scores"] if s > 0.5) / case_count - subset_avg_score = sum(data["scores"]) / case_count - subset_avg_duration = sum(data["durations"], timedelta()) / case_count - - print(f"{subset:<25} {case_count:<8} {subset_accuracy:<10.1%} {subset_avg_score:<10.3f} {str(subset_avg_duration):<12}") - - return { - "total_cases": total_cases, - "accuracy": accuracy, - "avg_score": avg_score, - "avg_duration": avg_duration, - "score_distribution": score_ranges, - "subset_stats": dict(subset_stats) - } +print(f"\nTotal results collected: {len(all_results):,}") -# Usage - analyze all results across all pages -analysis = analyze_evaluation_performance("eval_12345", use_all_pages=True) - -# Usage - analyze only first page (faster for quick checks) -quick_analysis = analyze_evaluation_performance("eval_12345", use_all_pages=False) +# Calculate accuracy +correct = sum(1 for r in all_results if r.score > 0.5) +accuracy = correct / len(all_results) if all_results else 0 +print(f"Overall accuracy: {accuracy:.1%} ({correct:,}/{len(all_results):,})") ``` -## Pagination Patterns - -### Pattern 1: Processing All Results Across Pages +### With Progress Tracking ```python from atlas import Atlas -def process_all_results(evaluation_id: str): - """Process all results by iterating through all pages""" +def get_all_results_with_progress(evaluation_id): client = Atlas() - - # Aggregate statistics across all pages - total_results = 0 - total_score = 0 - total_correct = 0 - all_subsets = set() - + all_results = [] page = 1 - page_size = 100 - - print("Processing all pages...") + page_size = 50 while True: print(f"Fetching page {page}...") - results_data = client.results.get( + results_data = client.results.get_by_id( evaluation_id=evaluation_id, page=page, page_size=page_size @@ -325,796 +96,170 @@ def process_all_results(evaluation_id: str): if not results_data or not results_data.results: break - # Show progress on first page - if page == 1: - print(f"Total: {results_data.pagination.total_count} results across {results_data.pagination.total_pages} pages") - - # Process current page - current_results = results_data.results - page_score = sum(r.score for r in current_results) - page_correct = sum(1 for r in current_results if r.score > 0.5) - page_subsets = set(r.subset for r in current_results) + all_results.extend(results_data.results) - # Aggregate - total_results += len(current_results) - total_score += page_score - total_correct += page_correct - all_subsets.update(page_subsets) + # Show progress + if page == 1: + total_count = results_data.pagination.total_count + total_pages = results_data.pagination.total_pages + print(f"Total results: {total_count:,} across {total_pages} pages") - print(f" Page {page}: {len(current_results)} results, {page_correct} correct, {len(page_subsets)} subsets") + progress = len(all_results) / results_data.pagination.total_count * 100 + print(f"Progress: {progress:.1f}% ({len(all_results):,} collected)") - # Check if we're done if page >= results_data.pagination.total_pages: break page += 1 - # Final summary - if total_results > 0: - overall_accuracy = total_correct / total_results - overall_avg_score = total_score / total_results - - print(f"\n Final Statistics:") - print(f" Total results processed: {total_results:,}") - print(f" Overall accuracy: {overall_accuracy:.1%}") - print(f" Overall average score: {overall_avg_score:.3f}") - print(f" Unique subsets: {len(all_subsets)}") - print(f" Subsets: {', '.join(sorted(all_subsets))}") - - return { - "total_results": total_results, - "accuracy": overall_accuracy if total_results > 0 else 0, - "avg_score": overall_avg_score if total_results > 0 else 0, - "subsets": list(all_subsets) - } + return all_results # Usage -stats = process_all_results("eval_12345") +results = get_all_results_with_progress("your_evaluation_id") +print(f"Done! Collected {len(results):,} results") ``` -### Pattern 2: Selective Page Processing +## Async Version ```python -def process_specific_pages(evaluation_id: str, start_page: int = 1, end_page: int = None): - """Process only specific pages of results""" - client = Atlas() - - # Get first page to understand scope - first_page = client.results.get(evaluation_id=evaluation_id, page=1, page_size=100) - if not first_page: - print(" No results found") - return None - - total_pages = first_page.pagination.total_pages - total_count = first_page.pagination.total_count - - # Set end page if not specified - if end_page is None: - end_page = total_pages - - # Validate range - end_page = min(end_page, total_pages) - start_page = max(start_page, 1) - - print(f" Processing pages {start_page}-{end_page} of {total_pages} (total: {total_count} results)") - - processed_results = [] - - for page_num in range(start_page, end_page + 1): - # Reuse first page if processing from page 1 - if page_num == 1 and start_page == 1: - results_data = first_page - else: - results_data = client.results.get( - evaluation_id=evaluation_id, - page=page_num, - page_size=100 - ) - - if not results_data: - print(f" Failed to get page {page_num}") - continue - - processed_results.extend(results_data.results) - print(f" Processed page {page_num}: {len(results_data.results)} results") - - print(f" Processed {len(processed_results)} results from pages {start_page}-{end_page}") - return processed_results +import asyncio +from atlas import AsyncAtlas -# Usage examples -first_100_results = process_specific_pages("eval_12345", start_page=1, end_page=1) -middle_pages = process_specific_pages("eval_12345", start_page=5, end_page=10) -last_few_pages = process_specific_pages("eval_12345", start_page=18, end_page=20) -``` - -### Pattern 3: Smart Pagination with Early Stopping - -```python -def analyze_with_early_stopping(evaluation_id: str, min_accuracy_threshold: float = 0.7): - """Stop processing if accuracy drops below threshold""" - client = Atlas() +async def get_results_async(evaluation_id): + client = AsyncAtlas() + all_results = [] page = 1 - page_size = 100 - total_processed = 0 - total_correct = 0 - - print(f"šŸŽÆ Processing until accuracy drops below {min_accuracy_threshold:.1%}") while True: - results_data = client.results.get( + results_data = await client.results.get_by_id( evaluation_id=evaluation_id, page=page, - page_size=page_size + page_size=100 ) if not results_data or not results_data.results: break - # Process current page - current_results = results_data.results - page_correct = sum(1 for r in current_results if r.score > 0.5) + all_results.extend(results_data.results) + print(f"Page {page}: {len(results_data.results)} results") - total_processed += len(current_results) - total_correct += page_correct - - current_accuracy = total_correct / total_processed - page_accuracy = page_correct / len(current_results) - - print(f" Page {page}: {page_accuracy:.1%} accuracy ({page_correct}/{len(current_results)})") - print(f" Running total: {current_accuracy:.1%} accuracy ({total_correct}/{total_processed})") - - # Check early stopping condition - if current_accuracy < min_accuracy_threshold and page > 1: - print(f" Stopping early: accuracy ({current_accuracy:.1%}) below threshold ({min_accuracy_threshold:.1%})") - break - - # Check if we've processed all pages if page >= results_data.pagination.total_pages: - print(f" Processed all {results_data.pagination.total_pages} pages") break page += 1 - final_accuracy = total_correct / total_processed if total_processed > 0 else 0 - print(f"\n Final Results:") - print(f" Pages processed: {page}/{results_data.pagination.total_pages if 'results_data' in locals() else '?'}") - print(f" Results processed: {total_processed}") - print(f" Final accuracy: {final_accuracy:.1%}") - - return { - "pages_processed": page, - "results_processed": total_processed, - "accuracy": final_accuracy, - "stopped_early": page < (results_data.pagination.total_pages if 'results_data' in locals() else 1) - } - -# Usage -early_stop_results = analyze_with_early_stopping("eval_12345", min_accuracy_threshold=0.8) -``` - -### Comparative Analysis - -```python -from atlas import Atlas -from typing import List, Dict - -def compare_evaluation_results(evaluation_ids: List[str], labels: List[str] = None): - """Compare results across multiple evaluations""" - client = Atlas() - - if labels and len(labels) != len(evaluation_ids): - labels = [f"Eval {i+1}" for i in range(len(evaluation_ids))] - elif not labels: - labels = [f"Eval {i+1}" for i in range(len(evaluation_ids))] - - print(f" Comparing {len(evaluation_ids)} evaluations") - print(f"{'='*80}") - - # Collect results for all evaluations - all_results = {} - for eval_id, label in zip(evaluation_ids, labels): - results = client.results.get(evaluation_id=eval_id) - if results: - all_results[label] = results - print(f" Loaded {len(results)} results for {label}") - else: - print(f" No results found for {label} ({eval_id})") - - if not all_results: - print(" No results to compare") - return - - print(f"\n Comparative Analysis:") - print(f"{'Metric':<20} " + " ".join(f"{label:<15}" for label in labels)) - print("-" * (20 + 15 * len(labels))) - - # Compare key metrics - metrics = {} - for label, results in all_results.items(): - total_cases = len(results) - correct_answers = sum(1 for r in results if r.score > 0.5) - accuracy = correct_answers / total_cases - avg_score = sum(r.score for r in results) / total_cases - avg_duration = sum(r.duration for r in results) / len(results) - - metrics[label] = { - "total_cases": total_cases, - "accuracy": accuracy, - "avg_score": avg_score, - "avg_duration": avg_duration - } - - # Print comparison table - print(f"{'Total Cases':<20} " + " ".join(f"{metrics[label]['total_cases']:<15,}" for label in labels)) - print(f"{'Accuracy':<20} " + " ".join(f"{metrics[label]['accuracy']:<15.1%}" for label in labels)) - print(f"{'Average Score':<20} " + " ".join(f"{metrics[label]['avg_score']:<15.3f}" for label in labels)) - print(f"{'Average Duration':<20} " + " ".join(f"{str(metrics[label]['avg_duration']):<15}" for label in labels)) - - # Find best performing evaluation - best_accuracy = max(metrics.values(), key=lambda x: x["accuracy"]) - best_speed = min(metrics.values(), key=lambda x: x["avg_duration"]) - - best_accuracy_label = next(label for label, data in metrics.items() if data == best_accuracy) - best_speed_label = next(label for label, data in metrics.items() if data == best_speed) - - print(f"\n Winners:") - print(f" Best Accuracy: {best_accuracy_label} ({best_accuracy['accuracy']:.1%})") - print(f" Fastest: {best_speed_label} ({best_speed['avg_duration']})") - - # Subset-level comparison (if results have same subsets) - if len(all_results) >= 2: - first_subsets = set(r.subset for r in list(all_results.values())[0]) - common_subsets = first_subsets - - for results in list(all_results.values())[1:]: - result_subsets = set(r.subset for r in results) - common_subsets = common_subsets.intersection(result_subsets) - - if common_subsets: - print(f"\n Subset Comparison ({len(common_subsets)} common subsets):") - print(f"{'Subset':<25} " + " ".join(f"{label} Acc":<12 for label in labels)) - print("-" * (25 + 12 * len(labels))) - - for subset in sorted(common_subsets): - subset_accuracies = [] - for label, results in all_results.items(): - subset_results = [r for r in results if r.subset == subset] - if subset_results: - subset_accuracy = sum(1 for r in subset_results if r.score > 0.5) / len(subset_results) - subset_accuracies.append(f"{subset_accuracy:.1%}") - else: - subset_accuracies.append("N/A") - - print(f"{subset:<25} " + " ".join(f"{acc:<12}" for acc in subset_accuracies)) - - return metrics + return all_results -# Usage - compare GPT-4 vs Claude-3 on MMLU -evaluation_ids = ["eval_gpt4_mmlu", "eval_claude3_mmlu", "eval_llama2_mmlu"] -labels = ["GPT-4", "Claude-3", "Llama-2"] - -comparison = compare_evaluation_results(evaluation_ids, labels) +# Run it +results = asyncio.run(get_results_async("your_evaluation_id")) +print(f"Total: {len(results)} results") ``` -### Error Analysis +## Complete Workflow ```python from atlas import Atlas -def analyze_failures(evaluation_id: str, error_threshold: float = 0.3): - """Analyze cases where the model performed poorly""" client = Atlas() - results = client.results.get(evaluation_id=evaluation_id) - if not results: - print(f" No results found for evaluation {evaluation_id}") - return None - - # Find poor-performing cases - poor_results = [r for r in results if r.score < error_threshold] - good_results = [r for r in results if r.score >= error_threshold] - - print(f" Error Analysis for {evaluation_id}") - print(f"{'='*60}") - print(f"Total cases: {len(results)}") - print(f"Poor performance (< {error_threshold}): {len(poor_results)} ({len(poor_results)/len(results):.1%})") - print(f"Good performance (>= {error_threshold}): {len(good_results)} ({len(good_results)/len(results):.1%})") - - if not poor_results: - print(" No poor-performing cases found!") - return {"poor_results": [], "analysis": "No errors to analyze"} - - # Analyze failure patterns by subset - failure_by_subset = {} - for result in poor_results: - if result.subset not in failure_by_subset: - failure_by_subset[result.subset] = [] - failure_by_subset[result.subset].append(result) - - print(f"\n Failure Distribution by Subset:") - for subset, failures in sorted(failure_by_subset.items(), key=lambda x: len(x[1]), reverse=True): - total_in_subset = len([r for r in results if r.subset == subset]) - failure_rate = len(failures) / total_in_subset - print(f" {subset}: {len(failures)}/{total_in_subset} failures ({failure_rate:.1%})") - - # Show worst-performing examples - worst_results = sorted(poor_results, key=lambda x: x.score)[:5] - - print(f"\n Worst Performing Examples:") - for i, result in enumerate(worst_results, 1): - print(f"\n Example {i} [Score: {result.score:.3f}]") - print(f" Subset: {result.subset}") - print(f" Prompt: {result.prompt[:200]}...") - print(f" Model Answer: {result.result[:100]}...") - print(f" Expected: {result.truth[:100]}...") - print(f" Duration: {result.duration}") - - if result.metrics: - print(f" Additional Metrics: {result.metrics}") - - # Common failure patterns - print(f"\n Common Patterns in Failures:") - - # Analyze prompt lengths - poor_prompt_lengths = [len(r.prompt) for r in poor_results] - good_prompt_lengths = [len(r.prompt) for r in good_results] - - avg_poor_prompt_len = sum(poor_prompt_lengths) / len(poor_prompt_lengths) - avg_good_prompt_len = sum(good_prompt_lengths) / len(good_prompt_lengths) - - print(f" Average prompt length in failures: {avg_poor_prompt_len:.0f} chars") - print(f" Average prompt length in successes: {avg_good_prompt_len:.0f} chars") - - # Analyze response lengths - poor_response_lengths = [len(r.result) for r in poor_results] - good_response_lengths = [len(r.result) for r in good_results] - - avg_poor_response_len = sum(poor_response_lengths) / len(poor_response_lengths) - avg_good_response_len = sum(good_response_lengths) / len(good_response_lengths) - - print(f" Average response length in failures: {avg_poor_response_len:.0f} chars") - print(f" Average response length in successes: {avg_good_response_len:.0f} chars") - - # Analyze durations - avg_poor_duration = sum(r.duration for r in poor_results) / len(poor_results) - avg_good_duration = sum(r.duration for r in good_results) / len(good_results) - - print(f" Average duration for failures: {avg_poor_duration}") - print(f" Average duration for successes: {avg_good_duration}") - - return { - "poor_results": poor_results, - "failure_by_subset": failure_by_subset, - "worst_examples": worst_results, - "patterns": { - "avg_poor_prompt_len": avg_poor_prompt_len, - "avg_good_prompt_len": avg_good_prompt_len, - "avg_poor_response_len": avg_poor_response_len, - "avg_good_response_len": avg_good_response_len, - "avg_poor_duration": avg_poor_duration, - "avg_good_duration": avg_good_duration - } - } - -# Usage -error_analysis = analyze_failures("eval_12345", error_threshold=0.5) -``` +# 1. Create evaluation +models = client.models.get() +benchmarks = client.benchmarks.get() -## Advanced Result Processing - -### Batch Processing Large Result Sets - -```python -from atlas import Atlas -from typing import Iterator, List -import time +evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] +) +print(f"Created evaluation: {evaluation.id}") + +# 2. Wait for completion +print("Waiting for evaluation to complete...") +completed_evaluation = client.evaluations.wait_for_completion( + evaluation, + interval_seconds=30, + timeout=1800 # 30 minutes +) -def process_results_in_batches(evaluation_id: str, batch_size: int = 100, processor_func=None): - """Process large result sets in manageable batches""" - client = Atlas() +# 3. Get all results +if completed_evaluation.is_success: + print("Getting results...") - results = client.results.get(evaluation_id=evaluation_id) - if not results: - print(f" No results found for evaluation {evaluation_id}") - return None - - total_results = len(results) - print(f" Processing {total_results:,} results in batches of {batch_size}") - - if not processor_func: - # Default processor: just count scores - def processor_func(batch): - return { - "count": len(batch), - "avg_score": sum(r.score for r in batch) / len(batch), - "correct": sum(1 for r in batch if r.score > 0.5) - } - - batch_results = [] + all_results = [] + page = 1 - for i in range(0, total_results, batch_size): - batch = results[i:i + batch_size] - batch_num = i // batch_size + 1 - total_batches = (total_results + batch_size - 1) // batch_size - - print(f" Processing batch {batch_num}/{total_batches} ({len(batch)} items)") - - start_time = time.time() - batch_result = processor_func(batch) - end_time = time.time() - - batch_result.update({ - "batch_num": batch_num, - "processing_time": end_time - start_time, - "items_processed": len(batch) - }) - - batch_results.append(batch_result) + while True: + results_data = client.results.get_by_id( + evaluation_id=completed_evaluation.id, + page=page, + page_size=100 + ) - print(f" Completed in {batch_result['processing_time']:.2f}s") + if not results_data or not results_data.results: + break - # Small delay to prevent overwhelming the system - if batch_num < total_batches: - time.sleep(0.1) - - # Aggregate results - total_processing_time = sum(br["processing_time"] for br in batch_results) - total_correct = sum(br.get("correct", 0) for br in batch_results) - overall_accuracy = total_correct / total_results - - print(f"\n Batch Processing Summary:") - print(f" Total batches: {len(batch_results)}") - print(f" Total processing time: {total_processing_time:.2f}s") - print(f" Average time per batch: {total_processing_time/len(batch_results):.2f}s") - print(f" Overall accuracy: {overall_accuracy:.1%}") - - return { - "batch_results": batch_results, - "summary": { - "total_items": total_results, - "total_batches": len(batch_results), - "total_processing_time": total_processing_time, - "overall_accuracy": overall_accuracy - } - } - -# Custom processor for subset analysis -def subset_analyzer(batch): - """Custom processor that analyzes subsets in a batch""" - subset_stats = {} + all_results.extend(results_data.results) + if page >= results_data.pagination.total_pages: + break + page += 1 - for result in batch: - if result.subset not in subset_stats: - subset_stats[result.subset] = {"count": 0, "total_score": 0, "correct": 0} - - subset_stats[result.subset]["count"] += 1 - subset_stats[result.subset]["total_score"] += result.score - if result.score > 0.5: - subset_stats[result.subset]["correct"] += 1 + # 4. Analyze results + correct = sum(1 for r in all_results if r.score > 0.5) + accuracy = correct / len(all_results) + avg_score = sum(r.score for r in all_results) / len(all_results) - return { - "subset_stats": subset_stats, - "unique_subsets": len(subset_stats) - } - -# Usage -batch_results = process_results_in_batches( - evaluation_id="eval_12345", - batch_size=50, - processor_func=subset_analyzer -) + print(f"Results: {len(all_results):,} total") + print(f"Accuracy: {accuracy:.1%}") + print(f"Average score: {avg_score:.3f}") +else: + print(f"Evaluation failed: {completed_evaluation.status}") ``` -### Result Caching and Persistence +## Analyze Results by Subset ```python -import json -import pickle -from pathlib import Path -from datetime import datetime from atlas import Atlas -import atlas +from collections import defaultdict -class ResultsCache: - """Cache evaluation results to avoid repeated API calls""" - - def __init__(self, cache_dir: str = "results_cache"): - self.cache_dir = Path(cache_dir) - self.cache_dir.mkdir(exist_ok=True) - - def _get_cache_path(self, evaluation_id: str, format: str = "json") -> Path: - """Get cache file path for an evaluation""" - return self.cache_dir / f"{evaluation_id}_results.{format}" - - def _get_metadata_path(self, evaluation_id: str) -> Path: - """Get metadata file path for an evaluation""" - return self.cache_dir / f"{evaluation_id}_metadata.json" - - def is_cached(self, evaluation_id: str) -> bool: - """Check if results are already cached""" - return self._get_cache_path(evaluation_id).exists() - - def save_results(self, evaluation_id: str, results: list, metadata: dict = None): - """Save results to cache""" - try: - # Save as JSON (human-readable) - json_path = self._get_cache_path(evaluation_id, "json") - with open(json_path, 'w') as f: - # Convert results to serializable format - serializable_results = [] - for result in results: - result_dict = { - "subset": result.subset, - "prompt": result.prompt, - "result": result.result, - "truth": result.truth, - "score": result.score, - "duration": str(result.duration), # Convert timedelta to string - "metrics": result.metrics - } - serializable_results.append(result_dict) - - json.dump(serializable_results, f, indent=2, ensure_ascii=False) - - # Save as pickle (preserves exact object types) - pickle_path = self._get_cache_path(evaluation_id, "pkl") - with open(pickle_path, 'wb') as f: - pickle.dump(results, f) - - # Save metadata - if not metadata: - metadata = {} - - metadata.update({ - "evaluation_id": evaluation_id, - "cached_at": datetime.now().isoformat(), - "result_count": len(results), - "cache_format": "both" - }) - - metadata_path = self._get_metadata_path(evaluation_id) - with open(metadata_path, 'w') as f: - json.dump(metadata, f, indent=2) - - print(f"šŸ’¾ Cached {len(results)} results for {evaluation_id}") - - except Exception as e: - print(f" Error caching results: {e}") - - def load_results(self, evaluation_id: str, format: str = "pickle"): - """Load results from cache""" - try: - if format == "pickle": - cache_path = self._get_cache_path(evaluation_id, "pkl") - with open(cache_path, 'rb') as f: - results = pickle.load(f) - else: - cache_path = self._get_cache_path(evaluation_id, "json") - with open(cache_path, 'r') as f: - results = json.load(f) - - print(f"šŸ’¾ Loaded {len(results)} results from cache for {evaluation_id}") - return results - - except Exception as e: - print(f" Error loading cached results: {e}") - return None - - def get_metadata(self, evaluation_id: str): - """Get cached metadata""" - try: - metadata_path = self._get_metadata_path(evaluation_id) - with open(metadata_path, 'r') as f: - return json.load(f) - except Exception as e: - print(f" Error loading metadata: {e}") - return None - -def get_results_with_cache(evaluation_id: str, cache: ResultsCache = None, force_refresh: bool = False): - """Get results with automatic caching""" - if not cache: - cache = ResultsCache() - - # Check cache first (unless force refresh) - if not force_refresh and cache.is_cached(evaluation_id): - print(f"šŸ“‚ Loading results from cache...") - cached_results = cache.load_results(evaluation_id) - - if cached_results: - metadata = cache.get_metadata(evaluation_id) - if metadata: - cached_at = metadata.get("cached_at", "unknown") - print(f"šŸ“… Cached at: {cached_at}") - return cached_results - - # Fetch from API - print(f"šŸŒ Fetching fresh results from API...") client = Atlas() - - try: - results = client.results.get(evaluation_id=evaluation_id) - - if results: - # Cache the results - cache.save_results(evaluation_id, results) - return results - else: - print(f" No results found for evaluation {evaluation_id}") - return None - - except atlas.APIError as e: - print(f" Error fetching results: {e}") - - # Try to return cached results as fallback - if cache.is_cached(evaluation_id): - print(f" Falling back to cached results...") - return cache.load_results(evaluation_id) - - return None - -# Usage examples -cache = ResultsCache("./my_results_cache") - -# First call - fetches from API and caches -results1 = get_results_with_cache("eval_12345", cache) - -# Second call - loads from cache -results2 = get_results_with_cache("eval_12345", cache) - -# Force refresh from API -results3 = get_results_with_cache("eval_12345", cache, force_refresh=True) - -# Batch cache multiple evaluations -evaluation_ids = ["eval_001", "eval_002", "eval_003"] - -for eval_id in evaluation_ids: - results = get_results_with_cache(eval_id, cache) - if results: - print(f" {eval_id}: {len(results)} results cached") - -print(f"\nšŸ“ Cache contents:") -for cache_file in cache.cache_dir.glob("*.json"): - if cache_file.name.endswith("_metadata.json"): - continue - evaluation_id = cache_file.stem.replace("_results", "") - metadata = cache.get_metadata(evaluation_id) - if metadata: - count = metadata.get("result_count", "unknown") - cached_at = metadata.get("cached_at", "unknown") - print(f" {evaluation_id}: {count} results (cached: {cached_at})") +evaluation_id = "your_evaluation_id" + +# Get all results +all_results = [] +page = 1 + +while True: + results_data = client.results.get_by_id(evaluation_id=evaluation_id, page=page) + if not results_data or not results_data.results: + break + all_results.extend(results_data.results) + if page >= results_data.pagination.total_pages: + break + page += 1 + +# Group by subset +subset_results = defaultdict(list) +for result in all_results: + subset_results[result.subset].append(result) + +# Analyze each subset +print(f"Analysis by subset:") +for subset, results in subset_results.items(): + correct = sum(1 for r in results if r.score > 0.5) + accuracy = correct / len(results) + avg_score = sum(r.score for r in results) / len(results) + + print(f" {subset}:") + print(f" Cases: {len(results)}") + print(f" Accuracy: {accuracy:.1%}") + print(f" Avg Score: {avg_score:.3f}") ``` -### Export and Reporting - -```python -import csv -from pathlib import Path -from datetime import datetime -from atlas import Atlas - -def export_results_to_csv(evaluation_id: str, output_path: str = None): - """Export evaluation results to CSV format""" - client = Atlas() - - results = client.results.get(evaluation_id=evaluation_id) - if not results: - print(f" No results found for evaluation {evaluation_id}") - return None - - if not output_path: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output_path = f"results_{evaluation_id}_{timestamp}.csv" - - try: - with open(output_path, 'w', newline='', encoding='utf-8') as csvfile: - fieldnames = [ - 'subset', 'prompt', 'model_response', 'expected_answer', - 'score', 'duration_ms', 'prompt_length', 'response_length' - ] - - # Add metric columns if they exist - if results and results[0].metrics: - metric_keys = list(results[0].metrics.keys()) - fieldnames.extend([f"metric_{key}" for key in metric_keys]) - - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - - for result in results: - row = { - 'subset': result.subset, - 'prompt': result.prompt, - 'model_response': result.result, - 'expected_answer': result.truth, - 'score': result.score, - 'duration_ms': int(result.duration.total_seconds() * 1000), - 'prompt_length': len(result.prompt), - 'response_length': len(result.result) - } - - # Add metrics if present - if result.metrics: - for key, value in result.metrics.items(): - row[f"metric_{key}"] = value - - writer.writerow(row) - - print(f" Exported {len(results)} results to {output_path}") - return output_path - - except Exception as e: - print(f" Error exporting to CSV: {e}") - return None - -def generate_summary_report(evaluation_ids: list, output_path: str = None): - """Generate a summary report comparing multiple evaluations""" - client = Atlas() - - if not output_path: - timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - output_path = f"evaluation_summary_{timestamp}.txt" - - with open(output_path, 'w') as f: - f.write("ATLAS EVALUATION SUMMARY REPORT\n") - f.write("=" * 50 + "\n") - f.write(f"Generated: {datetime.now().isoformat()}\n") - f.write(f"Evaluations analyzed: {len(evaluation_ids)}\n\n") - - for i, eval_id in enumerate(evaluation_ids, 1): - f.write(f"EVALUATION {i}: {eval_id}\n") - f.write("-" * 30 + "\n") - - results = client.results.get(evaluation_id=eval_id) - - if not results: - f.write(" No results found\n\n") - continue - - # Calculate statistics - total_cases = len(results) - correct_answers = sum(1 for r in results if r.score > 0.5) - accuracy = correct_answers / total_cases - avg_score = sum(r.score for r in results) / total_cases - avg_duration = sum(r.duration for r in results) / len(results) - - # Write statistics - f.write(f"Total test cases: {total_cases:,}\n") - f.write(f"Correct answers: {correct_answers:,}\n") - f.write(f"Accuracy: {accuracy:.1%}\n") - f.write(f"Average score: {avg_score:.3f}\n") - f.write(f"Average duration: {avg_duration}\n") - - # Subset breakdown - subset_stats = {} - for result in results: - if result.subset not in subset_stats: - subset_stats[result.subset] = [] - subset_stats[result.subset].append(result.score) - - f.write(f"\nSubset Performance:\n") - for subset, scores in sorted(subset_stats.items()): - subset_accuracy = sum(1 for s in scores if s > 0.5) / len(scores) - subset_avg = sum(scores) / len(scores) - f.write(f" {subset}: {subset_accuracy:.1%} accuracy, {subset_avg:.3f} avg score ({len(scores)} cases)\n") - - f.write("\n") - - f.write("END OF REPORT\n") - - print(f" Summary report generated: {output_path}") - return output_path - -# Usage examples +## Key Points -# Export single evaluation to CSV -csv_path = export_results_to_csv("eval_12345") - -# Generate summary report for multiple evaluations -evaluation_list = ["eval_gpt4_mmlu", "eval_claude3_mmlu", "eval_llama2_mmlu"] -report_path = generate_summary_report(evaluation_list) - -print(f"Files generated:") -print(f" CSV Export: {csv_path}") -print(f" Summary Report: {report_path}") -``` +- Results are paginated with default page size of 100 +- Always loop through all pages to get complete results +- Use `page_size` parameter to control how many results per page (1-500) +- Check `pagination.total_pages` to know when to stop +- Results contain `score`, `subset`, `prompt`, `result`, and `truth` fields +- Use async versions for better performance with large datasets diff --git a/docs/examples/timeouts.md b/docs/examples/timeouts.md index c635dc0..a40861c 100644 --- a/docs/examples/timeouts.md +++ b/docs/examples/timeouts.md @@ -1,715 +1,231 @@ # Working with Timeouts -This guide provides practical examples for configuring and handling timeouts effectively with the Atlas Python SDK. +Simple examples for configuring timeouts with the Atlas Python SDK. -## Understanding Timeouts +## Basic Timeouts -Timeouts in the Atlas SDK control how long to wait for API responses. Different operations may require different timeout configurations based on their expected duration and criticality. - -## Basic Timeout Configuration - -### Simple Timeout +### Set Client Timeout ```python from atlas import Atlas -# Set a 2-minute timeout for all requests -client = Atlas(timeout=120.0) +# Set timeout for all requests (in seconds) +client = Atlas(timeout=300.0) # 5 minutes -# Create evaluation with 2-minute timeout -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" -) +# Use the client normally +models = client.models.get() +evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) ``` -### Default Timeout Behavior +### Different Timeouts for Different Use Cases ```python from atlas import Atlas -# Uses default timeout (10 minutes) -client = Atlas() +# Quick operations (testing, health checks) +quick_client = Atlas(timeout=30.0) # 30 seconds -print(f"Default timeout: {client.timeout}") # Should show 10 minutes in seconds +# Normal operations +normal_client = Atlas(timeout=300.0) # 5 minutes + +# Long operations (large evaluations) +patient_client = Atlas(timeout=1800.0) # 30 minutes + +# Use appropriate client +models = quick_client.models.get() # Fast operation +evaluation = patient_client.evaluations.create(...) # Slow operation ``` -## Advanced Timeout Configuration +## Per-Request Timeouts -### Granular Timeout Control +### Override Timeout for Specific Requests ```python -import httpx from atlas import Atlas -# Configure different timeouts for different operations -client = Atlas( - timeout=httpx.Timeout( - connect=10.0, # 10 seconds to establish connection - read=300.0, # 5 minutes to read response - write=30.0, # 30 seconds to send request - pool=60.0 # 1 minute for connection pool operations - ) -) +client = Atlas(timeout=60.0) # Default 1 minute +# Override timeout for specific operations +models = client.models.get() +benchmarks = client.benchmarks.get() + +# This evaluation might take longer, so use extended timeout evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" + model=models[0], + benchmark=benchmarks[0], + timeout=1800.0 # 30 minutes for this specific request ) ``` -### Per-Request Timeout Override +## Async Timeouts ```python -from atlas import Atlas +import asyncio +from atlas import AsyncAtlas -# Client with default 1-minute timeout -client = Atlas(timeout=60.0) - -# Override timeout for specific operations -try: - # Quick operation with short timeout - quick_eval = client.with_options(timeout=30.0).evaluations.create( - model="gpt-3.5-turbo", - benchmark="arc-easy" - ) +async def main(): + # Set timeout for async client + client = await AsyncAtlas.create(timeout=300.0) - # Long operation with extended timeout - complex_eval = client.with_options(timeout=600.0).evaluations.create( - model="gpt-4", - benchmark="math" # Complex benchmark - ) + models = await client.models.get() + benchmarks = await client.benchmarks.get() - # Results retrieval with medium timeout - results = client.with_options(timeout=120.0).results.get( - evaluation_id=quick_eval.id + # Create evaluation with custom timeout + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0], + timeout=1200.0 # 20 minutes ) -except Exception as e: - print(f"Operation failed: {e}") -``` + return evaluation -## Timeout Strategies by Use Case +asyncio.run(main()) +``` -### Development and Testing +## Handling Timeout Errors ```python from atlas import Atlas import atlas -def development_client(): - """Client optimized for development with shorter timeouts""" - return Atlas( - timeout=30.0 # 30 seconds - fail fast during development - ) - -def test_api_connectivity(): - """Quick connectivity test with very short timeout""" - client = development_client() +def safe_operation_with_timeout(): + client = Atlas(timeout=60.0) # 1 minute timeout try: - # Use simple, fast operation to test connectivity - evaluation = client.with_options(timeout=10.0).evaluations.create( - model="gpt-3.5-turbo", # Usually faster - benchmark="arc-easy" # Smaller benchmark + models = client.models.get() + benchmarks = client.benchmarks.get() + + evaluation = client.evaluations.create( + model=models[0], + benchmark=benchmarks[0] ) - if evaluation: - print("āœ… API connectivity confirmed") - return True - else: - print("āŒ API returned no evaluation") - return False + return evaluation except atlas.APITimeoutError: - print("āŒ API timeout - connectivity issues or server overload") - return False - except atlas.APIConnectionError: - print("āŒ Connection failed - check network") - return False + print("Operation timed out") + print("Try increasing the timeout or check your connection") + return None + except atlas.APIError as e: - print(f"āŒ API error: {e}") - return False + print(f"Other API error: {e}") + return None -# Usage -if test_api_connectivity(): - print("Proceeding with full evaluation...") -else: - print("Fix connectivity issues before continuing") +result = safe_operation_with_timeout() ``` -### Production Workloads +## Wait for Completion with Timeout ```python -import httpx from atlas import Atlas -import atlas -def production_client(): - """Client optimized for production workloads""" - return Atlas( - timeout=httpx.Timeout( - connect=30.0, # 30s to connect (allows for network delays) - read=1800.0, # 30 minutes for complex evaluations - write=60.0, # 1 minute to send large requests - pool=120.0 # 2 minutes for connection pool - ) - ) - -def robust_evaluation_creation(model: str, benchmark: str, max_retries: int = 3): - """Production-ready evaluation creation with timeout handling""" - client = production_client() - - for attempt in range(max_retries): - try: - print(f"šŸ”„ Attempt {attempt + 1}/{max_retries}: Creating evaluation...") - - evaluation = client.evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - print(f"āœ… Success: {evaluation.id}") - return evaluation - else: - print("āŒ No evaluation returned") - - except atlas.APITimeoutError: - print(f"ā° Timeout on attempt {attempt + 1}") - if attempt < max_retries - 1: - # Increase timeout for retry - retry_timeout = 1800.0 + (attempt * 600.0) # Add 10 minutes per retry - print(f"šŸ”„ Retrying with extended timeout: {retry_timeout/60:.0f} minutes") - - try: - evaluation = client.with_options(timeout=retry_timeout).evaluations.create( - model=model, - benchmark=benchmark - ) - if evaluation: - print(f"āœ… Success on retry: {evaluation.id}") - return evaluation - except atlas.APITimeoutError: - print(f"ā° Extended timeout also failed") - continue - else: - print("āŒ All timeout retry attempts failed") - - except atlas.APIError as e: - print(f"āŒ API error: {e}") - break # Don't retry API errors - - return None - -# Usage -evaluation = robust_evaluation_creation("gpt-4", "mmlu") -``` +client = Atlas() +models = client.models.get() +benchmarks = client.benchmarks.get() -### Batch Operations +# Create evaluation +evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) -```python -from atlas import Atlas -import atlas -import time - -def batch_evaluations_with_adaptive_timeout(model_benchmark_pairs: list): - """Create multiple evaluations with adaptive timeout strategy""" - client = Atlas(timeout=120.0) # Start with 2-minute timeout - - results = [] - consecutive_timeouts = 0 - current_timeout = 120.0 +# Wait for completion with timeout +try: + completed = client.evaluations.wait_for_completion( + evaluation, + interval_seconds=30, # Check every 30 seconds + timeout=3600 # Give up after 1 hour + ) - for i, (model, benchmark) in enumerate(model_benchmark_pairs, 1): - print(f"\n[{i}/{len(model_benchmark_pairs)}] {model} + {benchmark}") - print(f"Current timeout: {current_timeout/60:.1f} minutes") - - try: - evaluation = client.with_options(timeout=current_timeout).evaluations.create( - model=model, - benchmark=benchmark - ) - - if evaluation: - results.append({ - "model": model, - "benchmark": benchmark, - "evaluation_id": evaluation.id, - "success": True, - "timeout_used": current_timeout - }) - print(f"āœ… Success: {evaluation.id}") - - # Reset timeout on success - consecutive_timeouts = 0 - current_timeout = max(120.0, current_timeout * 0.9) # Slightly reduce timeout - else: - results.append({ - "model": model, - "benchmark": benchmark, - "success": False, - "error": "no_evaluation_returned" - }) - - except atlas.APITimeoutError: - print(f"ā° Timeout after {current_timeout/60:.1f} minutes") - consecutive_timeouts += 1 - - results.append({ - "model": model, - "benchmark": benchmark, - "success": False, - "error": "timeout", - "timeout_used": current_timeout - }) - - # Increase timeout after consecutive timeouts - if consecutive_timeouts >= 2: - current_timeout = min(3600.0, current_timeout * 1.5) # Max 1 hour - print(f"šŸ”„ Increased timeout to {current_timeout/60:.1f} minutes") - consecutive_timeouts = 0 # Reset counter after adjustment - - except atlas.APIError as e: - print(f"āŒ API error: {e}") - results.append({ - "model": model, - "benchmark": benchmark, - "success": False, - "error": str(e) - }) + if completed and completed.is_success: + print("Evaluation completed successfully") + else: + print("Evaluation failed") - # Brief pause between requests - time.sleep(1.0) - - # Summary - successful = [r for r in results if r["success"]] - timeouts = [r for r in results if r.get("error") == "timeout"] - - print(f"\nšŸ“Š Batch Summary:") - print(f" Total requests: {len(results)}") - print(f" Successful: {len(successful)}") - print(f" Timeouts: {len(timeouts)}") - print(f" Other errors: {len(results) - len(successful) - len(timeouts)}") - - return results - -# Usage -pairs = [ - ("gpt-4", "mmlu"), - ("claude-3-opus", "hellaswag"), - ("llama-2-70b", "arc-challenge"), - ("gpt-3.5-turbo", "gsm8k"), -] - -batch_results = batch_evaluations_with_adaptive_timeout(pairs) +except atlas.APITimeoutError: + print("Evaluation did not complete within 1 hour") + print(f"Current status: {evaluation.status}") ``` -## Error Handling and Recovery +## Advanced Timeout Configuration -### Timeout-Specific Error Handling +### Granular Control ```python -import atlas +import httpx from atlas import Atlas -import time - -def handle_timeout_gracefully(operation_func, *args, **kwargs): - """Generic timeout handler for any Atlas operation""" - max_retries = 3 - base_timeout = 60.0 - - for attempt in range(max_retries): - # Calculate timeout for this attempt - attempt_timeout = base_timeout * (2 ** attempt) # Exponential increase - - print(f"šŸ”„ Attempt {attempt + 1}/{max_retries} (timeout: {attempt_timeout/60:.1f}min)") - - try: - result = operation_func(timeout=attempt_timeout, *args, **kwargs) - print(f"āœ… Operation succeeded on attempt {attempt + 1}") - return result - - except atlas.APITimeoutError: - print(f"ā° Timeout on attempt {attempt + 1}") - - if attempt == max_retries - 1: - print("āŒ All retry attempts exhausted") - raise - else: - wait_time = 5 * (attempt + 1) # Progressive wait - print(f"ā³ Waiting {wait_time}s before retry...") - time.sleep(wait_time) - - except atlas.APIError as e: - print(f"āŒ Non-timeout error: {e}") - raise # Don't retry non-timeout errors - -def create_evaluation_with_timeout_handling(model: str, benchmark: str): - """Wrapper function for evaluation creation""" - def operation_func(timeout, *args, **kwargs): - client = Atlas(timeout=timeout) - return client.evaluations.create(model=model, benchmark=benchmark) - - return handle_timeout_gracefully(operation_func) -def get_results_with_timeout_handling(evaluation_id: str): - """Wrapper function for results retrieval""" - def operation_func(timeout, *args, **kwargs): - client = Atlas(timeout=timeout) - return client.results.get(evaluation_id=evaluation_id) - - return handle_timeout_gracefully(operation_func) +# Configure different timeouts for different operations +client = Atlas( + timeout=httpx.Timeout( + connect=10.0, # 10 seconds to connect + read=600.0, # 10 minutes to read response + write=30.0, # 30 seconds to send request + pool=30.0 # 30 seconds for connection pool + ) +) -# Usage -try: - evaluation = create_evaluation_with_timeout_handling("gpt-4", "mmlu") - if evaluation: - results = get_results_with_timeout_handling(evaluation.id) - print(f"šŸ“Š Retrieved {len(results) if results else 0} results") - -except atlas.APITimeoutError: - print("āŒ Operation failed due to persistent timeouts") -except atlas.APIError as e: - print(f"āŒ Operation failed: {e}") +# Use normally +evaluation = client.evaluations.create(model=models[0], benchmark=benchmarks[0]) ``` -### Circuit Breaker Pattern +## Recommended Timeouts + +### By Operation Type ```python -import time -from enum import Enum -import atlas from atlas import Atlas -class CircuitState(Enum): - CLOSED = "closed" # Normal operation - OPEN = "open" # Failing, don't try - HALF_OPEN = "half_open" # Testing if recovered +# Quick operations (< 30 seconds) +quick_client = Atlas(timeout=30.0) +models = quick_client.models.get() +benchmarks = quick_client.benchmarks.get() -class TimeoutCircuitBreaker: - """Circuit breaker specifically for timeout management""" - - def __init__(self, - failure_threshold: int = 5, - timeout_threshold: float = 300.0, # 5 minutes - recovery_timeout: int = 60): # 1 minute - self.failure_threshold = failure_threshold - self.timeout_threshold = timeout_threshold - self.recovery_timeout = recovery_timeout - - self.failure_count = 0 - self.last_failure_time = None - self.state = CircuitState.CLOSED - self.current_timeout = 120.0 # Start with 2 minutes - - def call(self, func, *args, **kwargs): - """Execute function with circuit breaker protection""" - if self.state == CircuitState.OPEN: - if (time.time() - self.last_failure_time) < self.recovery_timeout: - raise atlas.APIConnectionError( - message="Circuit breaker is OPEN - too many recent timeouts" - ) - else: - self.state = CircuitState.HALF_OPEN - print("šŸ”„ Circuit breaker transitioning to HALF_OPEN") - - try: - # Use adaptive timeout - if 'timeout' not in kwargs: - kwargs['timeout'] = self.current_timeout - - print(f"šŸ”„ Calling function with {self.current_timeout/60:.1f}min timeout") - result = func(*args, **kwargs) - - # Success - reset circuit breaker - self.on_success() - return result - - except atlas.APITimeoutError as e: - self.on_timeout_failure() - raise - except atlas.APIError as e: - # Non-timeout API errors don't affect circuit state - raise - - def on_success(self): - """Handle successful operation""" - print("āœ… Circuit breaker: Operation succeeded") - self.failure_count = 0 - self.state = CircuitState.CLOSED - - # Gradually reduce timeout on success - self.current_timeout = max(60.0, self.current_timeout * 0.95) - - def on_timeout_failure(self): - """Handle timeout failure""" - self.failure_count += 1 - self.last_failure_time = time.time() - - print(f"ā° Circuit breaker: Timeout failure {self.failure_count}/{self.failure_threshold}") - - # Increase timeout for next attempt - self.current_timeout = min(self.timeout_threshold, self.current_timeout * 1.5) - - if self.failure_count >= self.failure_threshold: - self.state = CircuitState.OPEN - print("šŸ”“ Circuit breaker: OPEN - too many consecutive timeouts") - -# Usage with circuit breaker -def protected_atlas_operations(): - """Example of using circuit breaker with Atlas operations""" - breaker = TimeoutCircuitBreaker( - failure_threshold=3, - timeout_threshold=600.0, # Max 10 minutes - recovery_timeout=120 # 2 minute recovery time - ) - - def create_evaluation_protected(model: str, benchmark: str): - def operation(timeout): - client = Atlas(timeout=timeout) - return client.evaluations.create(model=model, benchmark=benchmark) - return breaker.call(operation) - - def get_results_protected(evaluation_id: str): - def operation(timeout): - client = Atlas(timeout=timeout) - return client.results.get(evaluation_id=evaluation_id) - return breaker.call(operation) - - # Test with multiple operations - operations = [ - ("gpt-4", "mmlu"), - ("claude-3-opus", "hellaswag"), - ("llama-2-70b", "gsm8k"), - ] - - successful_evaluations = [] - - for model, benchmark in operations: - try: - print(f"\nšŸ”„ Creating evaluation: {model} + {benchmark}") - evaluation = create_evaluation_protected(model, benchmark) - - if evaluation: - successful_evaluations.append(evaluation) - print(f"āœ… Success: {evaluation.id}") - - # Try to get results - print(f"šŸ”„ Getting results for {evaluation.id}") - results = get_results_protected(evaluation.id) - - if results: - print(f"šŸ“Š Retrieved {len(results)} results") - - except atlas.APIConnectionError as e: - if "Circuit breaker is OPEN" in str(e): - print("šŸ”“ Circuit breaker prevented operation") - print(f"ā³ Waiting {breaker.recovery_timeout}s for recovery...") - time.sleep(breaker.recovery_timeout) - else: - print(f"āŒ Connection error: {e}") - - except atlas.APITimeoutError: - print("ā° Timeout occurred - circuit breaker updated") - - except atlas.APIError as e: - print(f"āŒ API error: {e}") - - print(f"\nšŸ“ˆ Final Results:") - print(f" Circuit state: {breaker.state.value}") - print(f" Current timeout: {breaker.current_timeout/60:.1f} minutes") - print(f" Successful evaluations: {len(successful_evaluations)}") - - return successful_evaluations +# Normal operations (5 minutes) +normal_client = Atlas(timeout=300.0) +evaluation = normal_client.evaluations.create(...) -# Run protected operations -results = protected_atlas_operations() +# Long operations (30+ minutes) +long_client = Atlas(timeout=1800.0) +completed = long_client.evaluations.wait_for_completion(...) + +# Getting results (depends on size) +results_client = Atlas(timeout=600.0) # 10 minutes +results = results_client.results.get_by_id(...) ``` -## Monitoring and Metrics +## Production Best Practices -### Timeout Performance Tracking +### Adaptive Timeouts ```python -import time -from dataclasses import dataclass -from typing import List, Optional from atlas import Atlas import atlas -@dataclass -class TimeoutMetrics: - operation: str - model: str - benchmark: str - timeout_set: float - actual_duration: float - success: bool - error_type: Optional[str] = None - timestamp: float = None - - def __post_init__(self): - if self.timestamp is None: - self.timestamp = time.time() - -class TimeoutMonitor: - """Monitor and analyze timeout patterns""" +def create_evaluation_with_adaptive_timeout(model, benchmark): + """Try with increasing timeouts""" - def __init__(self): - self.metrics: List[TimeoutMetrics] = [] + timeouts = [60.0, 300.0, 900.0] # 1min, 5min, 15min - def record_operation(self, operation: str, model: str, benchmark: str, - timeout_set: float, start_time: float, success: bool, - error_type: str = None): - """Record an operation's timeout metrics""" - actual_duration = time.time() - start_time - - metric = TimeoutMetrics( - operation=operation, - model=model, - benchmark=benchmark, - timeout_set=timeout_set, - actual_duration=actual_duration, - success=success, - error_type=error_type - ) - - self.metrics.append(metric) - - print(f"šŸ“Š Recorded: {operation} took {actual_duration:.1f}s (timeout: {timeout_set:.1f}s)") - - def get_timeout_efficiency(self) -> dict: - """Analyze timeout efficiency""" - if not self.metrics: - return {} - - successful_ops = [m for m in self.metrics if m.success] - timeout_ops = [m for m in self.metrics if m.error_type == "timeout"] - - analysis = { - "total_operations": len(self.metrics), - "successful_operations": len(successful_ops), - "timeout_operations": len(timeout_ops), - "success_rate": len(successful_ops) / len(self.metrics), - "timeout_rate": len(timeout_ops) / len(self.metrics), - } - - if successful_ops: - avg_success_duration = sum(m.actual_duration for m in successful_ops) / len(successful_ops) - avg_success_timeout = sum(m.timeout_set for m in successful_ops) / len(successful_ops) + for timeout in timeouts: + try: + client = Atlas(timeout=timeout) - analysis.update({ - "avg_success_duration": avg_success_duration, - "avg_success_timeout_set": avg_success_timeout, - "timeout_efficiency": avg_success_duration / avg_success_timeout if avg_success_timeout > 0 else 0 - }) - - return analysis - - def suggest_optimal_timeouts(self) -> dict: - """Suggest optimal timeouts based on historical data""" - if not self.metrics: - return {"message": "No data available"} - - # Group by operation type - by_operation = {} - for metric in self.metrics: - if metric.success: # Only use successful operations - key = (metric.operation, metric.model, metric.benchmark) - if key not in by_operation: - by_operation[key] = [] - by_operation[key].append(metric.actual_duration) - - suggestions = {} - for (operation, model, benchmark), durations in by_operation.items(): - # Suggest timeout as 95th percentile + 50% buffer - durations.sort() - p95_index = int(len(durations) * 0.95) - p95_duration = durations[p95_index] if p95_index < len(durations) else durations[-1] - suggested_timeout = p95_duration * 1.5 # 50% buffer + print(f"Trying with {timeout}s timeout...") + evaluation = client.evaluations.create(model=model, benchmark=benchmark) - suggestions[f"{operation}_{model}_{benchmark}"] = { - "suggested_timeout": suggested_timeout, - "based_on_operations": len(durations), - "p95_actual_duration": p95_duration - } - - return suggestions - -def monitored_atlas_operations(): - """Example of Atlas operations with timeout monitoring""" - monitor = TimeoutMonitor() - client = Atlas() - - test_operations = [ - ("gpt-3.5-turbo", "arc-easy", 60.0), # Should be fast - ("gpt-4", "mmlu", 180.0), # Medium complexity - ("claude-3-opus", "math", 600.0), # Complex, longer timeout - ] - - for model, benchmark, timeout in test_operations: - print(f"\nšŸ”„ Testing {model} + {benchmark} (timeout: {timeout/60:.1f}min)") - - # Evaluation creation - start_time = time.time() - try: - evaluation = client.with_options(timeout=timeout).evaluations.create( - model=model, - benchmark=benchmark - ) + print(f"Success with {timeout}s timeout") + return evaluation - if evaluation: - monitor.record_operation("create_evaluation", model, benchmark, - timeout, start_time, True) - - # Results retrieval - start_time = time.time() - try: - results = client.with_options(timeout=timeout).results.get( - evaluation_id=evaluation.id - ) - - success = results is not None - monitor.record_operation("get_results", model, benchmark, - timeout, start_time, success, - None if success else "no_results") - - except atlas.APITimeoutError: - monitor.record_operation("get_results", model, benchmark, - timeout, start_time, False, "timeout") - except atlas.APIError as e: - monitor.record_operation("get_results", model, benchmark, - timeout, start_time, False, str(e)) - else: - monitor.record_operation("create_evaluation", model, benchmark, - timeout, start_time, False, "no_evaluation") - except atlas.APITimeoutError: - monitor.record_operation("create_evaluation", model, benchmark, - timeout, start_time, False, "timeout") + print(f"Timed out with {timeout}s timeout") + continue except atlas.APIError as e: - monitor.record_operation("create_evaluation", model, benchmark, - timeout, start_time, False, str(e)) - - # Analyze results - print(f"\nšŸ“Š Timeout Analysis:") - efficiency = monitor.get_timeout_efficiency() + print(f"API error (not timeout): {e}") + break - for key, value in efficiency.items(): - if isinstance(value, float): - print(f" {key}: {value:.2f}") - else: - print(f" {key}: {value}") - - print(f"\nšŸ’” Timeout Suggestions:") - suggestions = monitor.suggest_optimal_timeouts() - for operation, suggestion in suggestions.items(): - print(f" {operation}:") - print(f" Suggested timeout: {suggestion['suggested_timeout']:.0f}s") - print(f" Based on {suggestion['based_on_operations']} successful operations") - print(f" 95th percentile duration: {suggestion['p95_actual_duration']:.1f}s") - -# Run monitoring example -monitored_atlas_operations() + print("Failed with all timeout attempts") + return None + +# Usage +models = Atlas().models.get() +benchmarks = Atlas().benchmarks.get() +evaluation = create_evaluation_with_adaptive_timeout(models[0], benchmarks[0]) ``` diff --git a/examples/get_benchmarks.py b/examples/get_benchmarks.py new file mode 100644 index 0000000..bdbb9d1 --- /dev/null +++ b/examples/get_benchmarks.py @@ -0,0 +1,25 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Get benchmarks by name + benchmark_name = "mmlu" + benchmarks = await client.benchmarks.get(name=benchmark_name) + print(f"Found {len(benchmarks)} benchmarks with name {benchmark_name}") + print(benchmarks) + + # --- Get benchmarks by type + benchmark_type = "public" + benchmarks = await client.benchmarks.get(type=benchmark_type) + print(f"Found {len(benchmarks)} benchmarks with type {benchmark_type}") + print(benchmarks) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/get_evaluation.py b/examples/get_evaluation.py new file mode 100644 index 0000000..615e232 --- /dev/null +++ b/examples/get_evaluation.py @@ -0,0 +1,19 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Get evaluation by id + evaluation_id = "eval_123" + evaluation = await client.evaluations.get(evaluation_id) + print(f"Found evaluation {evaluation.id}") + print(evaluation) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/get_models.py b/examples/get_models.py new file mode 100644 index 0000000..83061d0 --- /dev/null +++ b/examples/get_models.py @@ -0,0 +1,37 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Get models by name + model_name = "gpt-4o" + models = await client.models.get(name=model_name) + print(f"Found {len(models)} models with name {model_name}") + print(models) + + # --- Get models by company + company_names = ["openai", "anthropic"] + models = await client.models.get(companies=company_names) + print(f"Found {len(models)} models with companies {company_names}") + print(models) + + # --- Get models by region + region_names = ["usa"] + models = await client.models.get(regions=region_names) + print(f"Found {len(models)} models with regions {region_names}") + print(models) + + # --- Get models by type + model_type = "public" + models = await client.models.get(type=model_type) + print(f"Found {len(models)} models with type {model_type}") + print(models) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/paginated_results.py b/examples/paginated_results.py new file mode 100644 index 0000000..1e300c6 --- /dev/null +++ b/examples/paginated_results.py @@ -0,0 +1,105 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get() + print(f"Found {len(models)} models") + + # --- Benchmarks + benchmarks = await client.benchmarks.get() + print(f"Found {len(benchmarks)} benchmarks") + + # --- Create evaluation + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0], + ) + print(f"Created evaluation {evaluation.id}, status={evaluation.status}") + + # --- Wait for completion + evaluation = await client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + # Keep in mind that the evaluation will take a while to complete, so you may want to increase the timeout + # or grab the evaluation id and check the status later + timeout=600, # 10 minutes + ) + print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") + + # --- Results with pagination + if evaluation.is_success: + print("Fetching all results with pagination...") + + all_results = [] + page = 1 + page_size = 50 + + while True: + print(f"Fetching page {page} (page size: {page_size})...") + + # Get results for current page + results_data = await client.results.get_by_id( + evaluation_id=evaluation.id, + page=page, + page_size=page_size + ) + + if not results_data or not results_data.results: + print("No more results to fetch") + break + + # Add current page results to our collection + all_results.extend(results_data.results) + + # Show progress + if page == 1: + total_count = results_data.pagination.total_count + total_pages = results_data.pagination.total_pages + print(f"Total results: {total_count:,}") + print(f"Total pages: {total_pages}") + + print(f"Page {page}: Retrieved {len(results_data.results)} results") + print(f"Running total: {len(all_results):,} results") + + # Check if we've reached the last page + if page >= results_data.pagination.total_pages: + print("Reached last page") + break + + page += 1 + + # Summary of all results + print(f"\n=== PAGINATION COMPLETE ===") + print(f"Total results collected: {len(all_results):,}") + + if all_results: + # Calculate some basic statistics + correct_answers = sum(1 for r in all_results if r.score > 0.5) + accuracy = correct_answers / len(all_results) + avg_score = sum(r.score for r in all_results) / len(all_results) + + print(f"Overall accuracy: {accuracy:.1%} ({correct_answers:,}/{len(all_results):,})") + print(f"Average score: {avg_score:.3f}") + + # Show a few example results + print(f"\nFirst 3 results:") + for i, result in enumerate(all_results[:3], 1): + print(f" {i}. Score: {result.score:.3f}, Subset: {result.subset}") + print(f" Prompt: {result.prompt[:100]}...") + print(f" Response: {result.result[:100]}...") + print() + + else: + print("Evaluation did not succeed, no results to show.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/atlas/resources/results/results.py b/src/atlas/resources/results/results.py index 09f8d0d..b970992 100644 --- a/src/atlas/resources/results/results.py +++ b/src/atlas/resources/results/results.py @@ -11,6 +11,7 @@ DEFAULT_PAGE = 1 DEFAULT_PAGE_SIZE = 100 +MAX_PAGE_SIZE = 500 class Results(SyncAPIResource): @@ -68,12 +69,11 @@ def get_by_id( """ params = {"evaluation_id": evaluation_id} - effective_page_size = page_size if page_size is not None else DEFAULT_PAGE_SIZE + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE effective_page = page if page is not None else DEFAULT_PAGE params["page"] = str(effective_page) - if page_size is not None: - params["pageSize"] = str(page_size) + params["pageSize"] = str(effective_page_size) # Get the response with cast_to to get parsed data resp = self._get( @@ -221,7 +221,7 @@ async def get_by_id( """ params = {"evaluation_id": evaluation_id} - effective_page_size = page_size if page_size is not None else DEFAULT_PAGE_SIZE + effective_page_size = min(max(page_size, 1), MAX_PAGE_SIZE) if page_size is not None else DEFAULT_PAGE_SIZE effective_page = page if page is not None else DEFAULT_PAGE params["page"] = str(effective_page) diff --git a/tests/resources/test_results.py b/tests/resources/test_results.py index a22d9ae..6dd9517 100644 --- a/tests/resources/test_results.py +++ b/tests/resources/test_results.py @@ -87,7 +87,7 @@ def test_get_results_request_parameters(self, results_resource, mock_results_res results_resource._get.assert_called_once_with( "/results", - params={"evaluation_id": "eval-456", "page": "1"}, + params={"evaluation_id": "eval-456", "page": "1", "pageSize": "100"}, timeout=DEFAULT_TIMEOUT, cast_to=dict, ) @@ -642,7 +642,7 @@ def test_get_results_default_page_parameter(self, results_resource, sample_resul call_args = results_resource._get.call_args params = call_args.kwargs["params"] assert params["page"] == "1" - assert "pageSize" not in params # pageSize should not be included when not specified + assert params["pageSize"] == "100" # pageSize is now always included with default value def test_get_results_pagination_metadata_calculation(self, results_resource, sample_result_data): """get method correctly calculates pagination metadata.""" @@ -829,8 +829,8 @@ def test_get_results_zero_page_size_edge_case(self, results_resource): result = results_resource.get_by_id(evaluation_id="eval-123", page_size=0) assert isinstance(result, ResultsResponse) - # Should use 0 as provided (though this might cause division by zero, it's handled) - assert result.pagination.page_size == 0 + # page_size of 0 should be corrected to minimum value of 1 + assert result.pagination.page_size == 1 def test_get_results_negative_page_values(self, results_resource): """get method handles negative page values.""" @@ -854,9 +854,9 @@ def test_get_results_negative_page_values(self, results_resource): call_args = results_resource._get.call_args params = call_args.kwargs["params"] assert params["page"] == "-1" - assert params["pageSize"] == "-50" + assert params["pageSize"] == "1" # negative page_size should be corrected to 1 assert isinstance(result, ResultsResponse) - assert result.pagination.page_size == -50 - # total_pages calculation with negative page_size - assert result.pagination.total_pages == 0 # math.ceil handles negative divisors + assert result.pagination.page_size == 1 # negative page_size should be corrected to 1 + # total_pages calculation with corrected page_size + assert result.pagination.total_pages == 100 # math.ceil(100/1) = 100 From d51beab80082779c66b5c83d2aafbb557a90b6f8 Mon Sep 17 00:00:00 2001 From: Leandro Echevarria Date: Thu, 21 Aug 2025 11:28:05 -0300 Subject: [PATCH 2/2] feat | LAY-907 Updated docs and added examples to retrieve all results without manual pagination --- docs/examples/creating-evaluations.md | 2 +- docs/examples/retrieving-results.md | 58 ++++++++++++++++++++++++++- docs/examples/timeouts.md | 2 +- examples/all_results_no_pagination.py | 44 ++++++++++++++++++++ examples/async_client_simple.py | 2 +- examples/paginated_results.py | 2 +- 6 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 examples/all_results_no_pagination.py diff --git a/docs/examples/creating-evaluations.md b/docs/examples/creating-evaluations.md index e77c9fb..ff4a935 100644 --- a/docs/examples/creating-evaluations.md +++ b/docs/examples/creating-evaluations.md @@ -96,7 +96,7 @@ print(f"Created: {evaluation.id}") completed_evaluation = client.evaluations.wait_for_completion( evaluation, interval_seconds=30, # Check every 30 seconds - timeout=1800 # 30 minute timeout + timeout_seconds=1800 # 30 minute timeout ) if completed_evaluation.is_success: diff --git a/docs/examples/retrieving-results.md b/docs/examples/retrieving-results.md index 06f9b64..c25615c 100644 --- a/docs/examples/retrieving-results.md +++ b/docs/examples/retrieving-results.md @@ -32,7 +32,61 @@ else: print("No results found") ``` -## Pagination - Get All Results +## Get All Results Without Manual Pagination + +### Using get_all() Method + +The easiest way to retrieve all results is using the `get_all()` method, which handles pagination automatically: + +```python +import asyncio +from atlas import AsyncAtlas + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get() + print(f"Found {len(models)} models") + + # --- Benchmarks + benchmarks = await client.benchmarks.get() + print(f"Found {len(benchmarks)} benchmarks") + + # --- Create evaluation + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0], + ) + print(f"Created evaluation {evaluation.id}, status={evaluation.status}") + + # --- Wait for completion + evaluation = await client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + # Keep in mind that the evaluation will take a while to complete, so you may want to increase the timeout + # or grab the evaluation id and check the status later + timeout_seconds=600, # 10 minutes + ) + print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") + + # --- All results at once without pagination + results = await client.results.get_all(evaluation=evaluation) + print(f"Found {len(results)} results") + print(results) + +if __name__ == "__main__": + asyncio.run(main()) +``` + +This approach is much simpler than manual pagination as it: +- Automatically handles all pagination internally +- Returns all results in a single call +- No need to manage page numbers or loop through pages +- Works with both sync and async clients + +## Pagination - Get All Results (Manual) ### Simple Approach @@ -178,7 +232,7 @@ print("Waiting for evaluation to complete...") completed_evaluation = client.evaluations.wait_for_completion( evaluation, interval_seconds=30, - timeout=1800 # 30 minutes + timeout_seconds=1800 # 30 minutes ) # 3. Get all results diff --git a/docs/examples/timeouts.md b/docs/examples/timeouts.md index a40861c..6717512 100644 --- a/docs/examples/timeouts.md +++ b/docs/examples/timeouts.md @@ -131,7 +131,7 @@ try: completed = client.evaluations.wait_for_completion( evaluation, interval_seconds=30, # Check every 30 seconds - timeout=3600 # Give up after 1 hour + timeout_seconds=3600 # Give up after 1 hour ) if completed and completed.is_success: diff --git a/examples/all_results_no_pagination.py b/examples/all_results_no_pagination.py new file mode 100644 index 0000000..efc93d3 --- /dev/null +++ b/examples/all_results_no_pagination.py @@ -0,0 +1,44 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get() + print(f"Found {len(models)} models") + + # --- Benchmarks + benchmarks = await client.benchmarks.get() + print(f"Found {len(benchmarks)} benchmarks") + + # --- Create evaluation + evaluation = await client.evaluations.create( + model=models[0], + benchmark=benchmarks[0], + ) + print(f"Created evaluation {evaluation.id}, status={evaluation.status}") + + # --- Wait for completion + evaluation = await client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + # Keep in mind that the evaluation will take a while to complete, so you may want to increase the timeout + # or grab the evaluation id and check the status later + timeout_seconds=600, # 10 minutes + ) + print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") + + # --- All results at once without pagination + results = await client.results.get_all(evaluation=evaluation) + print(f"Found {len(results)} results") + print(results) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/async_client_simple.py b/examples/async_client_simple.py index 30db56a..f6e081c 100644 --- a/examples/async_client_simple.py +++ b/examples/async_client_simple.py @@ -23,7 +23,7 @@ async def main(): print(f"Created evaluation {evaluation.id}, status={evaluation.status}") # --- Wait for completion - await evaluation.wait_for_completion_async(interval_seconds=10, timeout=600) + await evaluation.wait_for_completion_async(interval_seconds=10, timeout_seconds=600) print(f"Evaluation {evaluation.id} finished with status={evaluation.status}") # --- Results diff --git a/examples/paginated_results.py b/examples/paginated_results.py index 1e300c6..52e6cf9 100644 --- a/examples/paginated_results.py +++ b/examples/paginated_results.py @@ -30,7 +30,7 @@ async def main(): interval_seconds=10, # Keep in mind that the evaluation will take a while to complete, so you may want to increase the timeout # or grab the evaluation id and check the status later - timeout=600, # 10 minutes + timeout_seconds=600, # 10 minutes ) print(f"Evaluation {evaluation.id} finished with status={evaluation.status}")