diff --git a/docs/api-reference/evaluations.md b/docs/api-reference/evaluations.md index 328c29e..81649af 100644 --- a/docs/api-reference/evaluations.md +++ b/docs/api-reference/evaluations.md @@ -55,6 +55,72 @@ evaluation = client.evaluations.create( ) ``` +### `get_by_id(evaluation_id, timeout=None)` + +Retrieves an existing evaluation by its unique identifier. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `evaluation_id` | `str` | Yes | The unique evaluation identifier | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns an `Evaluation` object if found, `None` if the evaluation does not exist or cannot be accessed. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Retrieve an evaluation by ID +evaluation_id = "eval_abc123xyz" +evaluation = client.evaluations.get_by_id(evaluation_id) + +if evaluation: + print(f"Found evaluation: {evaluation.id}") + print(f"Status: {evaluation.status}") + print(f"Model: {evaluation.model_name}") + print(f"Benchmark: {evaluation.dataset_name}") +else: + print(f"Evaluation {evaluation_id} not found") +``` + +#### With Custom Timeout + +```python +# Retrieve evaluation with custom timeout +evaluation = client.evaluations.get_by_id( + "eval_abc123xyz", + timeout=30.0 # 30 seconds +) +``` + +#### Async Usage + +```python +from atlas import AsyncAtlas +import asyncio + +async def get_evaluation(): + client = AsyncAtlas() + + evaluation = await client.evaluations.get_by_id("eval_abc123xyz") + if evaluation: + print(f"Found evaluation: {evaluation.id}") + return evaluation + else: + print("Evaluation not found") + return None + +# Run the async function +asyncio.run(get_evaluation()) +``` + ## Response Object The `create` method returns an `Evaluation` object with the following properties: @@ -167,31 +233,6 @@ if __name__ == "__main__": evaluation = create_and_monitor_evaluation() ``` -## Available Models - -Common model identifiers include: - -- `"gpt-4"` - OpenAI GPT-4 -- `"gpt-3.5-turbo"` - OpenAI GPT-3.5 Turbo -- `"claude-3-opus"` - Anthropic Claude 3 Opus -- `"claude-3-sonnet"` - Anthropic Claude 3 Sonnet -- `"llama-2-70b"` - Meta Llama 2 70B -- `"mistral-7b"` - Mistral 7B - -> **Note**: Available models may vary based on your organization's access. Check the LayerLens Atlas dashboard for the complete list of available models. - -## Available Benchmarks - -Common benchmark identifiers include: - -- `"mmlu"` - Massive Multitask Language Understanding -- `"hellaswag"` - HellaSwag commonsense reasoning -- `"arc-challenge"` - AI2 Reasoning Challenge -- `"truthfulqa"` - TruthfulQA -- `"winogrande"` - WinoGrande -- `"gsm8k"` - Grade School Math 8K - -> **Note**: Available benchmarks may vary based on your organization's access. Check the LayerLens Atlas dashboard for the complete list of available benchmarks. ## Error Handling @@ -267,15 +308,7 @@ evaluation = client.evaluations.create( ) ``` -### 3. Store Evaluation IDs -```python -# ✅ Good - store evaluation ID for later retrieval -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") -if evaluation: - # Store this ID in your database/system - evaluation_id = evaluation.id - print(f"Store this ID: {evaluation_id}") -``` + ## Next Steps diff --git a/docs/api-reference/models-benchmarks.md b/docs/api-reference/models-benchmarks.md index ccefdb1..7baacfa 100644 --- a/docs/api-reference/models-benchmarks.md +++ b/docs/api-reference/models-benchmarks.md @@ -8,10 +8,124 @@ Atlas evaluations require two key components: - **Model**: The AI model you want to evaluate - **Benchmark**: The dataset/test suite to evaluate the model against -The availability of models and benchmarks depends on your organization's access level and the specific Atlas deployment you're using. +The availability of models and benchmarks depends on your organizations configuration. + +### Finding Available Models and Benchmarks + +#### Check the Atlas Dashboard +The most reliable way to find available models and benchmarks: + +1. Log into your Atlas dashboard +2. Navigate to the evaluation creation page +3. View dropdown lists of available models and benchmarks + ## Models +### `get(type=None, name=None, companies=None, regions=None, licenses=None, timeout=None)` + +Retrieves a list of available models with optional filtering parameters. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `type` | `Literal["custom", "public"] \| None` | No | Filter by model type. If `None`, returns both custom and public models | +| `name` | `str \| None` | No | Filter models by name (partial match search) | +| `companies` | `List[str] \| None` | No | Filter by model companies/providers | +| `regions` | `List[str] \| None` | No | Filter by supported regions | +| `licenses` | `List[str] \| None` | No | Filter by license types | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `List[Model]` containing available models that match the filter criteria. Returns `None` if no models are found or if there's an error. + +#### Examples + +##### Get All Models + +```python +from atlas import Atlas + +client = Atlas() + +# Get all available models (both custom and public) +models = client.models.get() + +if models: + print(f"Found {len(models)} models:") + for model in models: + print(f" {model.id} - {model.name} ({model.company})") +else: + print("No models available") +``` + +##### Filter by Company + +```python +# Get models from specific companies +openai_models = client.models.get(companies=["OpenAI"]) +anthropic_models = client.models.get(companies=["Anthropic"]) + +# Get models from multiple companies +major_models = client.models.get(companies=["OpenAI", "Anthropic", "Google"]) + +if major_models: + for model in major_models: + print(f"{model.name} by {model.company}") +``` + +##### Search by Name + +```python +# Search for models with "gpt" in the name +gpt_models = client.models.get(name=["gpt"]) + + +if gpt_models: + print("GPT models found:") + for model in gpt_models: + print(f" {model.id} - {model.name}") +``` + +#### Model Object Properties + +Each `Model` object in the returned list contains: + +| Property | Type | Description | +|----------|------|-------------| +| `id` | `str` | Unique model identifier for use in evaluations | +| `name` | `str` | Human-readable model name | +| `company` | `str` | Company/provider that created the model | +| `type` | `str` | Model type ("custom" or "public") | +| `region` | `str` | Supported region | +| `license` | `str` | License type | + +#### Error Handling + +```python +import atlas +from atlas import Atlas + +client = Atlas() + +try: + models = client.models.get() + if models: + print(f"Retrieved {len(models)} models") + else: + print("No models available or error occurred") +except atlas.AuthenticationError: + print("Authentication failed - check your API key") +except atlas.PermissionDeniedError: + print("No permission to access models") +except atlas.APIConnectionError as e: + print(f"Connection error: {e}") +except atlas.APIError as e: + print(f"API error: {e}") +``` + ### Model Identification Models are identified by string IDs that you pass to the `evaluations.create()` method: @@ -44,57 +158,87 @@ if evaluation: ## Benchmarks -### Benchmark Identification +### `get(type=None, name=None, timeout=None)` -Benchmarks are identified by string IDs representing different evaluation datasets: +Retrieves a list of available benchmarks with optional filtering parameters. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `type` | `Literal["custom", "public"] \| None` | No | Filter by benchmark type. If `None`, returns both custom and public benchmarks | +| `name` | `str \| None` | No | Filter benchmarks by name (partial match search) | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `List[Benchmark]` containing available benchmarks that match the filter criteria. Returns `None` if no benchmarks are found or if there's an error. + +#### Examples + +##### Get All Benchmarks ```python from atlas import Atlas client = Atlas() -evaluation = client.evaluations.create( - model="gpt-4", - benchmark="mmlu" # Benchmark ID -) -``` +# Get all available benchmarks (both custom and public) +benchmarks = client.benchmarks.get() -### Benchmark Information +if benchmarks: + print(f"Found {len(benchmarks)} benchmarks:") + for benchmark in benchmarks: + print(f" {benchmark.id} - {benchmark.name}") +else: + print("No benchmarks available") +``` -Evaluation responses include benchmark details: +##### Search by Name ```python -evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") +# Search for benchmarks with "mmlu" in the name +mmlu_benchmark = client.benchmarks.get(name=["mmlu"])[0] -if evaluation: - print(f"Dataset ID: {evaluation.dataset_id}") # "mmlu" - print(f"Dataset Name: {evaluation.dataset_name}") # "MMLU" +if mmlu_benchmark: + print("MMLU benchmark found:") + print(f" {mmlu_benchmark.id} - {mmlu_benchmark.name}") ``` -### Performance Expectations -Different model-benchmark combinations yield different types of insights: +#### Benchmark Object Properties -#### General Intelligence Assessment -```python -# Broad capability assessment -models = ["gpt-4", "claude-3-opus", "llama-2-70b"] -benchmark = "mmlu" +Each `Benchmark` object in the returned list contains: -for model in models: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - # Compare general intelligence across models -``` +| Property | Type | Description | +|----------|------|-------------| +| `id` | `str` | Unique benchmark identifier for use in evaluations | +| `name` | `str` | Human-readable benchmark name | +| `type` | `str` | Benchmark type ("custom" or "public") | +| `description` | `str` | Description of what the benchmark tests | + +#### Error Handling -#### Specialized Task Performance ```python -# Code generation comparison -models = ["gpt-4", "code-llama-34b", "claude-3-sonnet"] -benchmark = "humaneval" +import atlas +from atlas import Atlas + +client = Atlas() -for model in models: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - # Compare coding abilities +try: + benchmarks = client.benchmarks.get() + if benchmarks: + print(f"Retrieved {len(benchmarks)} benchmarks") + else: + print("No benchmarks available or error occurred") +except atlas.AuthenticationError: + print("Authentication failed - check your API key") +except atlas.PermissionDeniedError: + print("No permission to access benchmarks") +except atlas.APIConnectionError as e: + print(f"Connection error: {e}") +except atlas.APIError as e: + print(f"API error: {e}") ``` ## Discovery and Validation @@ -107,173 +251,36 @@ The most reliable way to find available models and benchmarks: 1. Log into your Atlas dashboard 2. Navigate to the evaluation creation page 3. View dropdown lists of available models and benchmarks -4. Note the exact IDs for use in your code -#### Programmatic Discovery -While the SDK doesn't currently provide discovery endpoints, you can validate model/benchmark existence: +### Benchmark Identification + +Benchmarks are identified by string IDs representing different evaluation datasets: ```python -import atlas from atlas import Atlas -def validate_model_benchmark(model_id: str, benchmark_id: str) -> bool: - """Test if a model/benchmark combination is available""" - client = Atlas() - - try: - evaluation = client.evaluations.create( - model=model_id, - benchmark=benchmark_id - ) - - if evaluation: - print(f"✅ Valid: {model_id} + {benchmark_id}") - return True - else: - print(f"❌ Invalid: {model_id} + {benchmark_id}") - return False - - except atlas.NotFoundError: - print(f"❌ Not found: {model_id} or {benchmark_id}") - return False - except atlas.PermissionDeniedError: - print(f"❌ No access: {model_id} or {benchmark_id}") - return False - except atlas.APIError as e: - print(f"❌ Error: {e}") - return False - -# Test combinations -combinations = [ - ("gpt-4", "mmlu"), - ("claude-3-opus", "hellaswag"), - ("llama-2-70b", "arc-challenge"), - ("nonexistent-model", "mmlu"), # Should fail -] - -for model, benchmark in combinations: - validate_model_benchmark(model, benchmark) -``` - -### Batch Validation +client = Atlas() -```python -def batch_validate_combinations(model_benchmark_pairs): - """Validate multiple model/benchmark combinations""" - client = Atlas() - results = {} - - for model, benchmark in model_benchmark_pairs: - try: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - results[(model, benchmark)] = { - "valid": evaluation is not None, - "evaluation_id": evaluation.id if evaluation else None, - "model_name": evaluation.model_name if evaluation else None, - "dataset_name": evaluation.dataset_name if evaluation else None, - } - except atlas.APIError as e: - results[(model, benchmark)] = { - "valid": False, - "error": str(e), - "error_type": type(e).__name__ - } - - return results - -# Example usage -combinations = [ - ("gpt-4", "mmlu"), - ("claude-3-sonnet", "hellaswag"), - ("llama-2-70b", "gsm8k"), -] - -results = batch_validate_combinations(combinations) -for (model, benchmark), result in results.items(): - status = "✅" if result["valid"] else "❌" - print(f"{status} {model} + {benchmark}: {result}") +evaluation = client.evaluations.create( + model="gpt-4", + benchmark="mmlu" # Benchmark ID +) ``` -### Validate Before Production Use - -```python -def safe_create_evaluation(model: str, benchmark: str): - """Create evaluation with validation and error handling""" - client = Atlas() - - # Validate combination first - if not validate_model_benchmark(model, benchmark): - return None - - try: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - - if evaluation: - print(f"✅ Evaluation created successfully:") - print(f" ID: {evaluation.id}") - print(f" Model: {evaluation.model_name} ({evaluation.model_company})") - print(f" Benchmark: {evaluation.dataset_name}") - return evaluation - else: - print(f"❌ Failed to create evaluation") - return None - - except atlas.APIError as e: - print(f"❌ API error: {e}") - return None - -# Usage -evaluation = safe_create_evaluation("gpt-4", "mmlu") -``` +### Benchmark Information -### 4. Document Model and Benchmark Choices +Evaluation responses include benchmark details: ```python -# Document your evaluation strategy -EVALUATION_CONFIGS = { - "general_intelligence": { - "models": ["gpt-4", "claude-3-opus", "gemini-pro"], - "benchmarks": ["mmlu", "arc-challenge", "hellaswag"], - "description": "Broad cognitive ability assessment" - }, - "code_generation": { - "models": ["gpt-4", "code-llama-34b", "claude-3-sonnet"], - "benchmarks": ["humaneval", "mbpp", "apps"], - "description": "Programming and code generation capabilities" - }, - "mathematical_reasoning": { - "models": ["gpt-4", "claude-3-opus", "minerva-62b"], - "benchmarks": ["gsm8k", "math", "minerva-math"], - "description": "Mathematical problem-solving abilities" - } -} - -def run_evaluation_suite(suite_name: str): - """Run a predefined evaluation suite""" - if suite_name not in EVALUATION_CONFIGS: - print(f"Unknown suite: {suite_name}") - return - - config = EVALUATION_CONFIGS[suite_name] - print(f"Running {suite_name}: {config['description']}") - - client = Atlas() - evaluations = [] - - for model in config["models"]: - for benchmark in config["benchmarks"]: - evaluation = client.evaluations.create(model=model, benchmark=benchmark) - if evaluation: - evaluations.append(evaluation) - print(f"✅ {model} + {benchmark}: {evaluation.id}") - - return evaluations - -# Run comprehensive evaluation -evaluations = run_evaluation_suite("general_intelligence") +evaluation = client.evaluations.create(model="gpt-4", benchmark="mmlu") + +if evaluation: + print(f"Dataset ID: {evaluation.dataset_id}") # "mmlu" + print(f"Dataset Name: {evaluation.dataset_name}") # "MMLU" ``` + ## Troubleshooting ### Model or Benchmark Not Found diff --git a/docs/api-reference/results.md b/docs/api-reference/results.md index 28dca26..ae02de1 100644 --- a/docs/api-reference/results.md +++ b/docs/api-reference/results.md @@ -8,6 +8,136 @@ Results contain detailed information about each test case in an evaluation, incl ## Methods +### `get_all(evaluation, timeout=None)` + +Retrieves all results for a specific evaluation by automatically iterating through all pages. This is a convenience method that handles pagination internally. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `evaluation` | `Evaluation` | Yes | The evaluation object to get results for | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `List[Result]` containing all result objects across all pages. Returns an empty list if no results are found. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get evaluation first +evaluation = client.evaluations.get_by_id("eval_12345") +if not evaluation: + print("Evaluation not found") +else: + # Get all results at once + all_results = client.results.get_all(evaluation=evaluation) + + print(f"Retrieved {len(all_results)} total results") +``` + +#### Async Usage + +```python +from atlas import AsyncAtlas +import asyncio + +async def get_all_results(): + client = AsyncAtlas() + + # Get evaluation first + evaluation = await client.evaluations.get_by_id("eval_12345") + if not evaluation: + print("Evaluation not found") + return + + # Get all results asynchronously + all_results = await client.results.get_all(evaluation=evaluation) + print(f"Retrieved {len(all_results)} total results asynchronously") + + return all_results + +# Run the async function +asyncio.run(get_all_results()) +``` + +### `get_all_by_id(evaluation_id, timeout=None)` + +Retrieves all results for a specific evaluation by evaluation ID, automatically iterating through all pages. This is a convenience method that handles pagination internally. + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `evaluation_id` | `str` | Yes | The evaluation identifier to get results for | +| `timeout` | `float \| httpx.Timeout \| None` | No | Override request timeout | + +#### Returns + +Returns a `List[Result]` containing all result objects across all pages. Returns an empty list if no results are found or the evaluation doesn't exist. + +#### Example + +```python +from atlas import Atlas + +client = Atlas() + +# Get all results directly by evaluation ID +all_results = client.results.get_all_by_id(evaluation_id="eval_12345") + +if all_results: + print(f"Retrieved {len(all_results)} total results") + + # Calculate overall statistics + total_score = sum(result.score for result in all_results) + avg_score = total_score / len(all_results) + correct_count = sum(1 for result in all_results if result.score > 0.5) + accuracy = correct_count / len(all_results) + + print(f"Overall accuracy: {accuracy:.1%}") + print(f"Average score: {avg_score:.3f}") +else: + print("No results found for evaluation") +``` + +#### With Custom Timeout + +```python +# Get all results with custom timeout (5 minutes) +all_results = client.results.get_all_by_id( + evaluation_id="eval_12345", + timeout=300.0 +) +``` + +#### Async Usage + +```python +from atlas import AsyncAtlas +import asyncio + +async def get_all_results(): + client = AsyncAtlas() + + # Get all results asynchronously + all_results = await client.results.get_all_by_id(evaluation_id="eval_12345") + + if all_results: + print(f"Retrieved {len(all_results)} total results") + + else: + print("No results found") + +# Run the async function +asyncio.run(get_all_results()) +``` + ### `get(evaluation_id, page=None, page_size=None, timeout=None)` Retrieves detailed results for a specific evaluation with optional pagination support. @@ -182,95 +312,47 @@ Each `Result` object contains the following properties: - **`duration`**: Response latency as a Python `timedelta` object - **`metrics`**: Additional scoring metrics that may be benchmark-specific -## Complete Example +## Working with Large Result Sets + +### Fetching results async +Results can contain thousands of individual test cases. Consider using the async client to load results asynchronously: ```python -import atlas -from atlas import Atlas -from datetime import timedelta +import asyncio +from atlas import AsyncAtlas -def analyze_evaluation_results(evaluation_id: str): - client = Atlas() +async def fetch_results_efficiently(): + client = AsyncAtlas() - try: - # Get results - results_data = client.results.get(evaluation_id=evaluation_id) - - if not results_data: - print(f"No results found for evaluation {evaluation_id}") - return - - results = results_data.results - print(f"Analysis for evaluation {evaluation_id}") - print(f"Total test cases: {results_data.pagination.total_count}") - print(f"Results on current page: {len(results)}") - - # Calculate overall statistics for current page - total_score = sum(result.score for result in results) - avg_score = total_score / len(results) - correct_answers = sum(1 for result in results if result.score > 0.5) - accuracy = correct_answers / len(results) - - # Calculate timing statistics - durations = [result.duration for result in results] - avg_duration = sum(durations, timedelta()) / len(durations) - min_duration = min(durations) - max_duration = max(durations) - - print(f"\n🎯 Performance Metrics:") - print(f" Average Score: {avg_score:.3f}") - print(f" Accuracy: {accuracy:.1%} ({correct_answers}/{len(results)})") - print(f" Average Duration: {avg_duration}") - print(f" Min Duration: {min_duration}") - print(f" Max Duration: {max_duration}") - - # Group by subset - subset_stats = {} - for result in results: - if result.subset not in subset_stats: - subset_stats[result.subset] = {"scores": [], "count": 0} - subset_stats[result.subset]["scores"].append(result.score) - subset_stats[result.subset]["count"] += 1 - - print(f"\nPerformance by Subset:") - for subset, stats in subset_stats.items(): - subset_avg = sum(stats["scores"]) / len(stats["scores"]) - subset_acc = sum(1 for s in stats["scores"] if s > 0.5) / len(stats["scores"]) - print(f" {subset}: {subset_acc:.1%} accuracy ({subset_avg:.3f} avg score, {stats['count']} cases)") - - # Show some example results - print(f"\nSample Results:") - for i, result in enumerate(results[:3]): - status = "Correct" if result.score > 0.5 else "Incorrect" - print(f"\n Example {i+1} [{result.subset}] - {status}") - print(f" Prompt: {result.prompt[:100]}...") - print(f" Model Answer: {result.result[:100]}...") - print(f" Expected: {result.truth[:100]}...") - print(f" Score: {result.score}, Duration: {result.duration}") - - if result.metrics: - print(f" Additional Metrics: {result.metrics}") - - return results - - except atlas.NotFoundError: - print(f"Evaluation {evaluation_id} not found") - except atlas.AuthenticationError: - print("Authentication failed - check your API key") - except atlas.APIConnectionError as e: - print(f"Connection error: {e}") - except atlas.APIError as e: - print(f"API error: {e}") + # Get evaluation first + evaluation = await client.evaluations.get_by_id("eval_12345") + if not evaluation: + print("Evaluation not found") + return None - return None + # Good - async fetch with size awareness + results = await client.results.get_all(evaluation=evaluation) + if results: + print(f"Retrieved {len(results)} results asynchronously") + return results + else: + print("No results available") + return None -if __name__ == "__main__": - # Example usage - evaluation_id = "eval_12345" # Replace with actual evaluation ID - results = analyze_evaluation_results(evaluation_id) +# Run the async function +asyncio.run(fetch_results_efficiently()) ``` -## Working with Large Result Sets +```python +# Bad - synchronous blocking call for large datasets +from atlas import Atlas + +client = Atlas() +results = client.results.get(evaluation_id="eval_12345") +# This blocks the entire thread while fetching thousands of results +``` + +### Manually pagination through results For evaluations with many test cases, use pagination to efficiently process results: @@ -323,65 +405,46 @@ def process_results_efficiently(evaluation_id: str, page_size: int = 100): process_results_efficiently("eval_12345", page_size=50) ``` -### Memory-Efficient Processing +## Performance Considerations -The pagination approach is more memory-efficient than loading all results at once: +### Large Result Sets +Results can contain thousands of individual test cases. Consider using the async client to load results asynchronously: ```python -# Good - Memory efficient with pagination -def analyze_large_evaluation(evaluation_id: str): - client = Atlas() - - # Aggregate statistics across pages - total_processed = 0 - total_score = 0 - total_correct = 0 - - page = 1 - page_size = 100 - - while True: - results_data = client.results.get( - evaluation_id=evaluation_id, - page=page, - page_size=page_size - ) - - if not results_data or not results_data.results: - break - - # Process current page - page_score = sum(r.score for r in results_data.results) - page_correct = sum(1 for r in results_data.results if r.score > 0.5) - - total_score += page_score - total_correct += page_correct - total_processed += len(results_data.results) - - print(f"Page {page}: {len(results_data.results)} results, {page_correct} correct") - - # Check if we're done - if page >= results_data.pagination.total_pages: - break - - page += 1 +import asyncio +from atlas import AsyncAtlas + +async def fetch_results_efficiently(): + client = AsyncAtlas() - # Final statistics - overall_accuracy = total_correct / total_processed if total_processed > 0 else 0 - overall_avg_score = total_score / total_processed if total_processed > 0 else 0 + # Get evaluation first + evaluation = await client.evaluations.get_by_id("eval_12345") + if not evaluation: + print("Evaluation not found") + return None - print(f"\nFinal Results:") - print(f" Total processed: {total_processed}") - print(f" Overall accuracy: {overall_accuracy:.1%}") - print(f" Overall average score: {overall_avg_score:.3f}") - -# Bad - Loads everything into memory at once (may cause issues with large datasets) -def analyze_evaluation_inefficient(evaluation_id: str): - results_data = client.results.get(evaluation_id=evaluation_id) # No pagination - # This could load thousands of results into memory - for result in results_data.results: - # Process all results at once - pass + # Good - async fetch with size awareness + results = await client.results.get_all(evaluation=evaluation) + if results: + print(f"Retrieved {len(results)} results asynchronously") + if len(results) > 1000: + print("Large result set - consider processing in chunks") + return results + else: + print("No results available") + return None + +# Run the async function +asyncio.run(fetch_results_efficiently()) +``` + +```python +# Bad - synchronous blocking call for large datasets +from atlas import Atlas + +client = Atlas() +results = client.results.get(evaluation_id="eval_12345") +# This blocks the entire thread while fetching thousands of results ``` ## Filtering and Analysis @@ -479,50 +542,6 @@ def safe_get_results(client, evaluation_id): return [] ``` -## Performance Considerations - -### Large Result Sets -Results can contain thousands of individual test cases. Consider: - -```python -# Good - check result size first -results = client.results.get(evaluation_id="eval_12345") -if results: - print(f"Retrieved {len(results)} results") - if len(results) > 1000: - print("Large result set - consider processing in chunks") - -# Bad - not considering memory usage -results = client.results.get(evaluation_id="eval_12345") -# Process all results in memory without considering size -``` - -### Caching Results -For repeated analysis, consider caching results: - -```python -import pickle -from pathlib import Path - -def get_cached_results(client, evaluation_id, cache_dir="cache"): - cache_path = Path(cache_dir) / f"{evaluation_id}_results.pkl" - - if cache_path.exists(): - print("Loading cached results...") - with open(cache_path, 'rb') as f: - return pickle.load(f) - - print("Fetching fresh results...") - results = client.results.get(evaluation_id=evaluation_id) - - if results: - cache_path.parent.mkdir(exist_ok=True) - with open(cache_path, 'wb') as f: - pickle.dump(results, f) - - return results -``` - ## Best Practices ### 1. Always Check for Results @@ -552,20 +571,6 @@ for result in results: # Could be thousands of results expensive_processing(result) ``` -### 3. Use Meaningful Analysis -```python -# Good - extract meaningful insights -subset_performance = {} -for result in results: - if result.subset not in subset_performance: - subset_performance[result.subset] = [] - subset_performance[result.subset].append(result.score) - -# Bad - just print raw data -for result in results: - print(result.score) # Not very useful -``` - ## Next Steps - Learn about [error handling](errors.md) for robust applications diff --git a/examples/async_run_evaluations.py b/examples/async_run_evaluations.py new file mode 100644 index 0000000..d71acff --- /dev/null +++ b/examples/async_run_evaluations.py @@ -0,0 +1,94 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def create_and_run_evaluation(client, model, benchmark, eval_number): + """Create and run a single evaluation, tracking progress.""" + try: + print(f"Starting evaluation #{eval_number}...") + + # Create evaluation + evaluation = await client.evaluations.create(model=model, benchmark=benchmark) + print(f"✓ Created evaluation #{eval_number}: {evaluation.id}, status={evaluation.status}") + + # Wait for completion + evaluation = await client.evaluations.wait_for_completion( + evaluation, + interval_seconds=10, + timeout_seconds=600 # 10 minutes + ) + print(f"✓ Evaluation #{eval_number} ({evaluation.id}) finished with status={evaluation.status}") + + # Get results if successful + if evaluation.is_success: + results = await client.results.get_all(evaluation=evaluation) + print(f"✓ Evaluation #{eval_number} completed with {len(results)} results") + return eval_number, evaluation.id, len(results), True + else: + print(f"✗ Evaluation #{eval_number} did not succeed") + return eval_number, evaluation.id, 0, False + + except Exception as e: + print(f"✗ Error in evaluation #{eval_number}: {e}") + return eval_number, None, 0, False + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # --- Models + models = await client.models.get() + print(f"Found {len(models)} models") + + # --- Benchmarks + benchmarks = await client.benchmarks.get() + print(f"Found {len(benchmarks)} benchmarks") + + # Use first model and benchmark for all evaluations + target_model = models[0] + target_benchmark = benchmarks[0] + + print(f"Using model: {target_model}") + print(f"Using benchmark: {target_benchmark}") + print("=" * 80) + + # Create 3 evaluation tasks + num_evaluations = 3 + print(f"Starting {num_evaluations} evaluations in parallel...") + + tasks = [ + create_and_run_evaluation(client, target_model, target_benchmark, i + 1) + for i in range(num_evaluations) + ] + + # Execute all evaluations concurrently + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Summary + print("=" * 80) + print("SUMMARY:") + successful = 0 + total_results = 0 + + for result in results: + if isinstance(result, Exception): + print(f"Exception occurred: {result}") + else: + eval_num, eval_id, result_count, success = result + if success: + successful += 1 + total_results += result_count + print(f"Evaluation #{eval_num} ({eval_id}): SUCCESS - {result_count} results") + else: + print(f"Evaluation #{eval_num} ({eval_id}): FAILED") + + print(f"\nOverall: {successful}/{num_evaluations} evaluations succeeded") + print(f"Total results collected: {total_results}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/fetch_results_async.py b/examples/fetch_results_async.py new file mode 100644 index 0000000..b434411 --- /dev/null +++ b/examples/fetch_results_async.py @@ -0,0 +1,57 @@ +#!/usr/bin/env -S poetry run python + +import asyncio + +from atlas import AsyncAtlas + + +async def fetch_evaluation_results(client, evaluation_id): + """Fetch results for a single evaluation and print when loaded.""" + try: + print(f"Fetching evaluation {evaluation_id}...") + evaluation = await client.evaluations.get_by_id(evaluation_id) + print(f"Found evaluation {evaluation.id}, status={evaluation.status}") + + # Get all results for this evaluation + results = await client.results.get_all(evaluation=evaluation) + print(f"Loaded {len(results)} results for evaluation {evaluation_id}") + print(f"Results for {evaluation_id}: {results}") + print("-" * 80) + + return evaluation_id, results + except Exception as e: + print(f"Error fetching evaluation {evaluation_id}: {e}") + return evaluation_id, None + + +async def main(): + # Construct async client + client = AsyncAtlas() + + # List of evaluation IDs to fetch exmple + + evaluation_ids = [ + "68a65a3de7ad047fb5d8e7d4", + "688a254c673f6b2835cc7278" + ] + + print(f"Starting async fetch for {len(evaluation_ids)} evaluations...") + print("=" * 80) + + # Create tasks for concurrent execution + tasks = [ + fetch_evaluation_results(client, eval_id) + for eval_id in evaluation_ids + ] + + # Execute all tasks concurrently and print results as they complete + results = await asyncio.gather(*tasks, return_exceptions=True) + + print("=" * 80) + print("Summary:") + successful = sum(1 for _, result in results if result is not None and not isinstance(result, Exception)) + print(f"Successfully fetched results for {successful}/{len(evaluation_ids)} evaluations") + + +if __name__ == "__main__": + asyncio.run(main())