api_documentation_research_study.py

#!/usr/bin/env python3
"""
Complete API Documentation Quality Research Study
Tests how documentation quality affects LLM code generation accuracy
"""

import asyncio
import json
import os
import time
from datetime import datetime
from typing import Dict, List, Optional, Tuple
from dataclasses import dataclass, asdict
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.FileHandler('research_study.log'),
        logging.StreamHandler()
    ]
)
logger = logging.getLogger(__name__)

@dataclass
class APIConfig:
    """Configuration for an API in the research study"""
    name: str
    domain: str
    quality_level: float
    auth_method: str
    documentation_url: str
    test_endpoint: str
    env_key: str
    auth_type: str  # 'query', 'header', 'path', 'basic', 'bearer'
    notes: Optional[str] = None

@dataclass
class ExperimentResult:
    """Result of a single code generation experiment"""
    api_name: str
    domain: str
    quality_level: float
    auth_method: str
    llm_model: str
    attempt_number: int
    timestamp: str
    
    # Documentation extraction
    documentation_extracted: bool
    documentation_length: int
    extraction_time_ms: int
    
    # Code generation
    code_generated: bool
    generation_time_ms: int
    generated_code: Optional[str]
    code_length: int
    
    # Code execution
    execution_status: str  # 'full_success', 'partial_success', 'syntax_error', 'logic_error', 'api_error'
    execution_time_ms: int
    api_response_code: Optional[int]
    error_message: Optional[str]
    
    # Quality metrics
    has_error_handling: bool
    has_authentication: bool
    has_proper_imports: bool
    complexity_score: int

class APIDocumentationResearchStudy:
    """Main research study controller"""
    
    def __init__(self):
        self.results: List[ExperimentResult] = []
        self.start_time = datetime.now()
        self.verified_apis = self.load_verified_apis()
        self.llm_models = ['claude', 'gpt', 'gemini']
        self.attempts_per_api_per_model = 1
        
        logger.info(f"🔬 Initializing API Documentation Quality Research Study")
        logger.info(f"📊 Testing {len(self.verified_apis)} APIs × {len(self.llm_models)} LLMs × {self.attempts_per_api_per_model} attempts")
        logger.info(f"🧪 Total experiments: {len(self.verified_apis) * len(self.llm_models) * self.attempts_per_api_per_model}")
        
    def load_verified_apis(self) -> List[APIConfig]:
        """Load the 15 verified APIs from authentication test results"""
        
        # Load authentication results to get verified APIs
        try:
            with open('api_authentication_results.json', 'r') as f:
                auth_results = json.load(f)
                
            successful_apis = [r for r in auth_results['results'] if r['status'] == 'success']
            logger.info(f"✅ Loaded {len(successful_apis)} verified APIs from authentication test")
            
        except FileNotFoundError:
            logger.error("❌ Authentication results not found. Please run api_authentication_test.py first")
            raise
            
        # Define API configurations with documentation URLs
        api_configs = [
            # Weather APIs
            APIConfig(
                name="OpenWeatherMap",
                domain="Weather", 
                quality_level=4.0,
                auth_method="API Key (Query)",
                documentation_url="https://openweathermap.org/api/one-call-3",
                test_endpoint="https://api.openweathermap.org/data/2.5/weather",
                env_key="OPENWEATHER_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="Visual Crossing",
                domain="Weather",
                quality_level=4.5,
                auth_method="API Key (Query)", 
                documentation_url="https://www.visualcrossing.com/resources/documentation/weather-api/timeline-weather-api/",
                test_endpoint="https://weather.visualcrossing.com/VisualCrossingWebServices/rest/services/timeline",
                env_key="VISUAL_CROSSING_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="WeatherBit",
                domain="Weather",
                quality_level=3.0,
                auth_method="API Key (Query)",
                documentation_url="https://www.weatherbit.io/api/weather-current",
                test_endpoint="https://api.weatherbit.io/v2.0/current",
                env_key="WEATHERBIT_KEY", 
                auth_type="query"
            ),
            
            # News APIs
            APIConfig(
                name="NewsAPI",
                domain="News",
                quality_level=4.0,
                auth_method="API Key (Header)",
                documentation_url="https://newsapi.org/docs/endpoints/top-headlines",
                test_endpoint="https://newsapi.org/v2/top-headlines",
                env_key="NEWS_API_KEY",
                auth_type="header"
            ),
            APIConfig(
                name="NewsData.io",
                domain="News", 
                quality_level=4.0,
                auth_method="API Key (Query)",
                documentation_url="https://newsdata.io/documentation/#latest_news",
                test_endpoint="https://newsdata.io/api/1/news",
                env_key="NEWSDATA_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="Guardian API",
                domain="News",
                quality_level=3.5,
                auth_method="API Key (Query)",
                documentation_url="https://open-platform.theguardian.com/documentation/",
                test_endpoint="https://content.guardianapis.com/search",
                env_key="GUARDIAN_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="Currents API",
                domain="News",
                quality_level=3.5,
                auth_method="API Key (Query)",
                documentation_url="https://currentsapi.services/en/docs/",
                test_endpoint="https://api.currentsapi.services/v1/latest-news",
                env_key="CURRENTS_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="Mediastack",
                domain="News",
                quality_level=2.5,
                auth_method="API Key (Query)",
                documentation_url="https://mediastack.com/documentation",
                test_endpoint="http://api.mediastack.com/v1/news",
                env_key="MEDIASTACK_API_KEY",
                auth_type="query"
            ),
            
            # Currency APIs
            APIConfig(
                name="ExchangeRate-API",
                domain="Currency",
                quality_level=4.0,
                auth_method="API Key (Path)",
                documentation_url="https://www.exchangerate-api.com/docs/overview",
                test_endpoint="https://v6.exchangerate-api.com/v6",
                env_key="EXCHANGERATE_API_KEY",
                auth_type="path"
            ),
            APIConfig(
                name="Fixer.io",
                domain="Currency",
                quality_level=3.0,
                auth_method="API Key (Query)",
                documentation_url="https://fixer.io/documentation",
                test_endpoint="http://data.fixer.io/api/latest",
                env_key="FIXER_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="CurrencyAPI",
                domain="Currency",
                quality_level=3.0,
                auth_method="API Key (Query)",
                documentation_url="https://currencyapi.com/docs",
                test_endpoint="https://api.currencyapi.com/v3/latest",
                env_key="CURRENCY_API_KEY",
                auth_type="query"
            ),
            APIConfig(
                name="CurrencyLayer",
                domain="Currency",
                quality_level=2.0,
                auth_method="API Key (Query)",
                documentation_url="https://currencylayer.com/documentation",
                test_endpoint="http://api.currencylayer.com/live",
                env_key="CURRENCYLAYER_API_KEY",
                auth_type="query"
            ),
            
            # User Management APIs
            APIConfig(
                name="Clerk",
                domain="User Management",
                quality_level=2.0,
                auth_method="Bearer Token",
                documentation_url="https://clerk.com/docs/reference/backend-api",
                test_endpoint="https://api.clerk.dev/v1/users",
                env_key="CLERK_API_KEY",
                auth_type="bearer"
            ),
            
            # File Storage APIs
            APIConfig(
                name="Uploadcare",
                domain="File Storage",
                quality_level=3.5,
                auth_method="Public + Secret Key",
                documentation_url="https://uploadcare.com/docs/api_reference/rest/",
                test_endpoint="https://api.uploadcare.com/files/",
                env_key="UPLOADCARE_PUBLIC_KEY",
                auth_type="basic"
            ),
            
            # Utility APIs
            APIConfig(
                name="IPGeolocation",
                domain="Utility",
                quality_level=3.0,
                auth_method="API Key (Query)",
                documentation_url="https://ipgeolocation.io/documentation/ip-geolocation-api.html",
                test_endpoint="https://api.ipgeolocation.io/ipgeo",
                env_key="IPGEOLOCATION_API_KEY",
                auth_type="query"
            )
        ]
        
        # Filter to only include APIs that passed authentication
        verified_names = {api['name'] for api in successful_apis}
        verified_configs = [config for config in api_configs if config.name in verified_names]
        
        logger.info(f"🎯 Configured {len(verified_configs)} APIs for research study")
        return verified_configs

    async def run_complete_study(self):
        """Execute the complete research study pipeline"""

        logger.info("🚀 Starting Complete API Documentation Quality Research Study")
        logger.info("=" * 80)

        # Import components
        from documentation_extractor import DocumentationExtractor
        from llm_code_generator import LLMCodeGenerator
        from code_execution_engine import CodeExecutionEngine

        # Initialize components
        doc_extractor = DocumentationExtractor()
        code_generator = LLMCodeGenerator()
        code_executor = CodeExecutionEngine()

        try:
            # Initialize all components
            await doc_extractor.initialize()
            await code_generator.initialize()

            logger.info(f"✅ All components initialized")
            logger.info(f"📊 Beginning experiments on {len(self.verified_apis)} APIs")

            # Pre-extract all documentation in parallel for speed
            logger.info("🚀 Pre-extracting all API documentation in parallel...")
            documentation_tasks = [
                doc_extractor.extract_documentation(api.name, api.documentation_url)
                for api in self.verified_apis
            ]
            documentation_results = await asyncio.gather(*documentation_tasks, return_exceptions=True)

            # Create documentation mapping
            api_docs = {}
            for i, (api, result) in enumerate(zip(self.verified_apis, documentation_results)):
                if isinstance(result, Exception):
                    logger.warning(f"⚠️ Failed to extract docs for {api.name}: {result}")
                    api_docs[api.name] = (False, "", 0)
                else:
                    api_docs[api.name] = result

            logger.info(f"✅ Pre-extracted documentation for {len([r for r in documentation_results if not isinstance(r, Exception)])} APIs")

            # Run experiments for each API
            total_experiments = len(self.verified_apis) * len(self.llm_models) * self.attempts_per_api_per_model
            current_experiment = 0

            for api_config in self.verified_apis:
                logger.info(f"\n🔬 Processing API: {api_config.name} ({api_config.domain})")
                logger.info(f"📊 Quality Level: {api_config.quality_level}/5.0")
                logger.info(f"🔗 Documentation: {api_config.documentation_url}")

                # Get pre-extracted documentation
                doc_success, documentation, doc_time = api_docs.get(api_config.name, (False, "", 0))

                if not doc_success:
                    logger.warning(f"⚠️ No documentation available for {api_config.name}")
                    continue

                # Generate and test code with each LLM model
                for model in self.llm_models:
                    for attempt in range(self.attempts_per_api_per_model):
                        current_experiment += 1

                        logger.info(f"\n[{current_experiment}/{total_experiments}] {api_config.name} - {model} - Attempt {attempt + 1}")

                        result = await self.run_single_experiment(
                            api_config, documentation, doc_time, model, attempt + 1,
                            code_generator, code_executor
                        )

                        self.results.append(result)

                        # Log immediate result
                        if result.execution_status == "full_success":
                            logger.info(f"✅ SUCCESS: {result.api_name} - {result.llm_model}")
                        else:
                            logger.info(f"❌ {result.execution_status.upper()}: {result.api_name} - {result.llm_model}")

                        # Brief pause between experiments
                        await asyncio.sleep(1)

            # Generate final analysis and report
            await self.generate_final_report()

        except Exception as e:
            logger.error(f"💥 Research study failed: {e}")
            raise

        finally:
            # Cleanup all components
            await doc_extractor.cleanup()
            await code_generator.cleanup()

        logger.info("🎉 Research study completed successfully!")

    async def run_single_experiment(self, api_config: APIConfig, documentation: str, doc_time: int,
                                   model: str, attempt: int, code_generator, code_executor) -> ExperimentResult:
        """Run a single code generation and testing experiment"""

        timestamp = datetime.now().isoformat()

        # Generate code
        code_success, generated_code, gen_time, model_used = await code_generator.generate_code(
            api_config.name, documentation, api_config.auth_type, api_config.env_key
        )

        if not code_success:
            return ExperimentResult(
                api_name=api_config.name,
                domain=api_config.domain,
                quality_level=api_config.quality_level,
                auth_method=api_config.auth_method,
                llm_model=model,
                attempt_number=attempt,
                timestamp=timestamp,
                documentation_extracted=True,
                documentation_length=len(documentation),
                extraction_time_ms=doc_time,
                code_generated=False,
                generation_time_ms=gen_time,
                generated_code=None,
                code_length=0,
                execution_status="generation_failed",
                execution_time_ms=0,
                api_response_code=None,
                error_message="Code generation failed",
                has_error_handling=False,
                has_authentication=False,
                has_proper_imports=False,
                complexity_score=0
            )

        # Execute and test code
        execution_status, exec_time, response_code, error_msg, quality_metrics = await code_executor.execute_code(
            api_config.name, generated_code, api_config.env_key
        )

        return ExperimentResult(
            api_name=api_config.name,
            domain=api_config.domain,
            quality_level=api_config.quality_level,
            auth_method=api_config.auth_method,
            llm_model=model_used,
            attempt_number=attempt,
            timestamp=timestamp,
            documentation_extracted=True,
            documentation_length=len(documentation),
            extraction_time_ms=doc_time,
            code_generated=True,
            generation_time_ms=gen_time,
            generated_code=generated_code,
            code_length=len(generated_code),
            execution_status=execution_status,
            execution_time_ms=exec_time,
            api_response_code=response_code,
            error_message=error_msg,
            has_error_handling=quality_metrics.get('has_error_handling', False),
            has_authentication=quality_metrics.get('has_authentication', False),
            has_proper_imports=quality_metrics.get('has_proper_imports', False),
            complexity_score=quality_metrics.get('complexity_score', 0)
        )

    async def generate_final_report(self):
        """Generate comprehensive research report with statistical analysis"""

        logger.info("\n📊 Generating Final Research Report")
        logger.info("=" * 60)

        # Save raw results
        results_data = [asdict(result) for result in self.results]

        with open('research_study_results.json', 'w') as f:
            json.dump({
                "study_metadata": {
                    "start_time": self.start_time.isoformat(),
                    "end_time": datetime.now().isoformat(),
                    "total_experiments": len(self.results),
                    "apis_tested": len(self.verified_apis),
                    "llm_models": self.llm_models,
                    "attempts_per_api_per_model": self.attempts_per_api_per_model
                },
                "results": results_data
            }, f, indent=2)

        # Generate statistical analysis
        await self.perform_statistical_analysis()

        # Generate comprehensive report
        await self.generate_research_report()

        logger.info("📄 Final research report generated successfully")

    async def perform_statistical_analysis(self):
        """Perform statistical analysis of results"""

        logger.info("📈 Performing Statistical Analysis")

        # Calculate success rates by quality level
        quality_analysis = {}
        domain_analysis = {}
        auth_analysis = {}
        model_analysis = {}

        for result in self.results:
            # Quality level analysis
            quality = result.quality_level
            if quality not in quality_analysis:
                quality_analysis[quality] = {"total": 0, "success": 0}
            quality_analysis[quality]["total"] += 1
            if result.execution_status == "full_success":
                quality_analysis[quality]["success"] += 1

            # Domain analysis
            domain = result.domain
            if domain not in domain_analysis:
                domain_analysis[domain] = {"total": 0, "success": 0}
            domain_analysis[domain]["total"] += 1
            if result.execution_status == "full_success":
                domain_analysis[domain]["success"] += 1

            # Authentication method analysis
            auth = result.auth_method
            if auth not in auth_analysis:
                auth_analysis[auth] = {"total": 0, "success": 0}
            auth_analysis[auth]["total"] += 1
            if result.execution_status == "full_success":
                auth_analysis[auth]["success"] += 1

            # LLM model analysis
            model = result.llm_model
            if model not in model_analysis:
                model_analysis[model] = {"total": 0, "success": 0}
            model_analysis[model]["total"] += 1
            if result.execution_status == "full_success":
                model_analysis[model]["success"] += 1

        # Calculate success rates
        for analysis in [quality_analysis, domain_analysis, auth_analysis, model_analysis]:
            for key in analysis:
                total = analysis[key]["total"]
                success = analysis[key]["success"]
                analysis[key]["success_rate"] = (success / total * 100) if total > 0 else 0

        # Save analysis results
        analysis_results = {
            "quality_level_analysis": quality_analysis,
            "domain_analysis": domain_analysis,
            "auth_method_analysis": auth_analysis,
            "llm_model_analysis": model_analysis
        }

        with open('statistical_analysis.json', 'w') as f:
            json.dump(analysis_results, f, indent=2)

        # Log key findings
        logger.info("🔍 Key Statistical Findings:")

        # Quality level correlation
        quality_rates = [(q, data["success_rate"]) for q, data in quality_analysis.items()]
        quality_rates.sort()
        logger.info(f"📊 Success rates by quality level:")
        for quality, rate in quality_rates:
            logger.info(f"   {quality}/5.0: {rate:.1f}%")

        # Best performing domain
        best_domain = max(domain_analysis.items(), key=lambda x: x[1]["success_rate"])
        logger.info(f"🏆 Best performing domain: {best_domain[0]} ({best_domain[1]['success_rate']:.1f}%)")

        # Best performing LLM
        best_model = max(model_analysis.items(), key=lambda x: x[1]["success_rate"])
        logger.info(f"🤖 Best performing LLM: {best_model[0]} ({best_model[1]['success_rate']:.1f}%)")

        return analysis_results

    async def generate_research_report(self):
        """Generate comprehensive research report"""

        # Calculate overall statistics
        total_experiments = len(self.results)
        successful_experiments = len([r for r in self.results if r.execution_status == "full_success"])
        overall_success_rate = (successful_experiments / total_experiments * 100) if total_experiments > 0 else 0

        # Load statistical analysis
        with open('statistical_analysis.json', 'r') as f:
            analysis = json.load(f)

        # Generate markdown report
        report = f"""# API Documentation Quality Research Study - Final Report

## Executive Summary

This study investigated the relationship between API documentation quality and LLM code generation accuracy using {len(self.verified_apis)} verified APIs across 6 domains with quality levels ranging from 2.0/5.0 to 4.5/5.0.

### Key Findings

- **Overall Success Rate**: {overall_success_rate:.1f}% ({successful_experiments}/{total_experiments} experiments)
- **APIs Tested**: {len(self.verified_apis)} authenticated APIs
- **LLM Models**: {len(self.llm_models)} different models (Claude, GPT, Gemini)
- **Total Experiments**: {total_experiments} code generation and testing cycles

## Methodology

### Experimental Design
1. **Documentation Extraction**: Automated browsing of API documentation pages
2. **Code Generation**: LLM-generated Python integration code using live documentation
3. **Real API Testing**: Execution against actual APIs with verified authentication
4. **Success Measurement**: Full success, partial success, syntax errors, logic errors

### Quality Levels Tested
"""

        # Add quality level breakdown
        quality_analysis = analysis["quality_level_analysis"]
        for quality in sorted(quality_analysis.keys()):
            data = quality_analysis[quality]
            report += f"- **{quality}/5.0**: {data['success_rate']:.1f}% success rate ({data['success']}/{data['total']} experiments)\n"

        report += f"""
## Results by Domain

"""

        # Add domain analysis
        domain_analysis = analysis["domain_analysis"]
        for domain, data in sorted(domain_analysis.items(), key=lambda x: x[1]["success_rate"], reverse=True):
            report += f"- **{domain}**: {data['success_rate']:.1f}% success rate ({data['success']}/{data['total']} experiments)\n"

        report += f"""
## Results by Authentication Method

"""

        # Add authentication analysis
        auth_analysis = analysis["auth_method_analysis"]
        for auth, data in sorted(auth_analysis.items(), key=lambda x: x[1]["success_rate"], reverse=True):
            report += f"- **{auth}**: {data['success_rate']:.1f}% success rate ({data['success']}/{data['total']} experiments)\n"

        report += f"""
## Results by LLM Model

"""

        # Add model analysis
        model_analysis = analysis["llm_model_analysis"]
        for model, data in sorted(model_analysis.items(), key=lambda x: x[1]["success_rate"], reverse=True):
            report += f"- **{model}**: {data['success_rate']:.1f}% success rate ({data['success']}/{data['total']} experiments)\n"

        # Add error analysis
        error_types = {}
        for result in self.results:
            status = result.execution_status
            if status not in error_types:
                error_types[status] = 0
            error_types[status] += 1

        report += f"""
## Error Analysis

"""

        for error_type, count in sorted(error_types.items(), key=lambda x: x[1], reverse=True):
            percentage = (count / total_experiments * 100)
            report += f"- **{error_type.replace('_', ' ').title()}**: {count} ({percentage:.1f}%)\n"

        report += f"""
## Conclusions and Recommendations

### Research Question: "How does API documentation quality affect LLM code generation accuracy?"

Based on the experimental results:

1. **Documentation Quality Impact**: [Analysis of correlation between quality levels and success rates]

2. **Domain-Specific Patterns**: [Insights about which API domains work best with LLMs]

3. **Authentication Complexity**: [Impact of different authentication methods on success rates]

4. **LLM Model Performance**: [Comparison of different LLM models for API integration]

### Recommendations for API Documentation

1. **High-Priority Elements**: [What documentation elements most improve LLM success]
2. **Common Failure Points**: [What causes LLMs to generate failing code]
3. **Best Practices**: [Specific recommendations for API documentation authors]

## Study Limitations

- Sample size: {len(self.verified_apis)} APIs
- LLM models tested: {len(self.llm_models)}
- Time period: Single study execution
- Authentication scope: Only tested working API keys

## Data Availability

- Raw results: `research_study_results.json`
- Statistical analysis: `statistical_analysis.json`
- Study logs: `research_study.log`

---

*Study completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}*
*Total execution time: {datetime.now() - self.start_time}*
"""

        # Save report
        with open('API_Documentation_Quality_Research_Report.md', 'w') as f:
            f.write(report)

        logger.info("📋 Comprehensive research report saved to: API_Documentation_Quality_Research_Report.md")

# Main execution
async def main():
    """Execute the complete research study"""
    study = APIDocumentationResearchStudy()
    await study.run_complete_study()

if __name__ == "__main__":
    print("🔬 API Documentation Quality Research Study")
    print("Testing how documentation quality affects LLM code generation accuracy")
    print("This comprehensive study will take 30-60 minutes to complete")
    print("\nPress Ctrl+C to stop at any time")

    try:
        asyncio.run(main())
    except KeyboardInterrupt:
        print("\n👋 Research study stopped by user")
    except Exception as e:
        print(f"\n💥 Fatal error: {e}")
        print("Please check the logs for detailed error information")