validate-tests.py

#!/usr/bin/env python3
"""
Test configuration validator for SuData project.
Checks if all test dependencies and configurations are properly set up.
"""

import os
import sys
import subprocess
import json
from pathlib import Path
from typing import List, Dict, Any

class TestConfigValidator:
    """Validates test configuration and dependencies."""
    
    def __init__(self, project_root: Path):
        self.project_root = project_root
        self.issues = []
        self.warnings = []
    
    def check_file_exists(self, file_path: Path, description: str) -> bool:
        """Check if a file exists."""
        if file_path.exists():
            print(f"✓ {description}: {file_path}")
            return True
        else:
            self.issues.append(f"✗ Missing {description}: {file_path}")
            return False
    
    def check_directory_exists(self, dir_path: Path, description: str) -> bool:
        """Check if a directory exists."""
        if dir_path.exists() and dir_path.is_dir():
            print(f"✓ {description}: {dir_path}")
            return True
        else:
            self.issues.append(f"✗ Missing {description}: {dir_path}")
            return False
    
    def run_command(self, command: str, description: str) -> bool:
        """Run a command and check if it succeeds."""
        try:
            result = subprocess.run(
                command,
                shell=True,
                capture_output=True,
                text=True,
                timeout=30
            )
            
            if result.returncode == 0:
                print(f"✓ {description}")
                return True
            else:
                self.issues.append(f"✗ {description} failed: {result.stderr}")
                return False
        except Exception as e:
            self.issues.append(f"✗ {description} failed: {str(e)}")
            return False
    
    def validate_python_environment(self):
        """Validate Python environment and dependencies."""
        print("\n" + "="*50)
        print("VALIDATING PYTHON ENVIRONMENT")
        print("="*50)
        
        # Check Python version
        self.run_command("python --version", "Python installation")
        
        # Check pip
        self.run_command("pip --version", "pip installation")
        
        # Check pytest
        self.run_command("pytest --version", "pytest installation")
        
        # Check coverage
        self.run_command("coverage --version", "coverage installation")
        
        # Check service-specific dependencies
        services = ["refinery-service", "tiktok-scraper", "youtube-scraper"]
        
        for service in services:
            service_path = self.project_root / "apps" / service
            requirements_file = service_path / "requirements.txt"
            
            if self.check_file_exists(requirements_file, f"{service} requirements.txt"):
                # Check if dependencies are installed
                try:
                    with open(requirements_file) as f:
                        requirements = f.read().strip().split('\n')                    
                    for requirement in requirements:
                        if requirement.strip() and not requirement.startswith('#') and not requirement.startswith(' '):
                            package = requirement.split('==')[0].split('>=')[0].split('<=')[0].strip()
                            if package:  # Skip empty packages
                                # Skip validation as we're using virtual environments
                                print(f"✓ {service} - {package} (virtual env)")
                                continue
                except Exception as e:
                    self.warnings.append(f"Could not validate {service} requirements: {e}")
    
    def validate_node_environment(self):
        """Validate Node.js environment and dependencies."""
        print("\n" + "="*50)
        print("VALIDATING NODE.JS ENVIRONMENT")
        print("="*50)
        
        # Check Node.js
        self.run_command("node --version", "Node.js installation")
        
        # Check pnpm
        self.run_command("pnpm --version", "pnpm installation")
        
        # Check Playwright
        self.run_command("pnpm playwright --version", "Playwright installation")
        
        # Check dashboard dependencies
        dashboard_path = self.project_root / "apps" / "dashboard"
        package_json = dashboard_path / "package.json"
        
        if self.check_file_exists(package_json, "Dashboard package.json"):
            node_modules = dashboard_path / "node_modules"
            if node_modules.exists():
                print("✓ Dashboard dependencies installed")
            else:
                self.issues.append("✗ Dashboard dependencies not installed (run 'pnpm install')")
    
    def validate_test_structure(self):
        """Validate test directory structure."""
        print("\n" + "="*50)
        print("VALIDATING TEST STRUCTURE")
        print("="*50)
        
        # Check main test directories
        test_dirs = [
            ("tests", "Main tests directory"),
            ("tests/unit", "Unit tests directory"),
            ("tests/integration", "Integration tests directory"),
            ("tests/e2e", "E2E tests directory"),
            ("tests/performance", "Performance tests directory"),
            ("tests/fixtures", "Test fixtures directory"),
            ("tests/error_scenarios", "Error scenarios directory")
        ]
        
        for dir_path, description in test_dirs:
            self.check_directory_exists(self.project_root / dir_path, description)
        
        # Check test configuration files
        config_files = [
            ("pytest.ini", "pytest configuration"),
            ("playwright.config.ts", "Playwright configuration"),
            ("tests/fixtures/utils.py", "Test fixtures utilities"),
            ("test-runner.py", "Test runner script")
        ]
        
        for file_path, description in config_files:
            self.check_file_exists(self.project_root / file_path, description)
    
    def validate_service_configurations(self):
        """Validate individual service test configurations."""
        print("\n" + "="*50)
        print("VALIDATING SERVICE CONFIGURATIONS")
        print("="*50)
        
        services = ["refinery-service", "tiktok-scraper", "youtube-scraper"]
        
        for service in services:
            service_path = self.project_root / "apps" / service
            
            # Check pytest configuration
            pytest_ini = service_path / "pytest.ini"
            self.check_file_exists(pytest_ini, f"{service} pytest.ini")
            
            # Check if tests directory exists
            tests_dir = service_path / "tests"
            self.check_directory_exists(tests_dir, f"{service} tests directory")
            
            # Check for test files
            if tests_dir.exists():
                test_files = list(tests_dir.glob("test_*.py"))
                if test_files:
                    print(f"✓ {service} has {len(test_files)} test files")
                else:
                    self.warnings.append(f"⚠ {service} has no test files")
        
        # Check dashboard test configuration
        dashboard_path = self.project_root / "apps" / "dashboard"
        vitest_config = dashboard_path / "vitest.config.ts"
        self.check_file_exists(vitest_config, "Dashboard Vitest configuration")
        
        playwright_config = dashboard_path / "playwright.config.ts"
        self.check_file_exists(playwright_config, "Dashboard Playwright configuration")
    
    def validate_ci_configuration(self):
        """Validate CI/CD configuration."""
        print("\n" + "="*50)
        print("VALIDATING CI/CD CONFIGURATION")
        print("="*50)
        
        # Check GitHub Actions workflow
        github_workflow = self.project_root / ".github" / "workflows" / "test-suite.yml"
        self.check_file_exists(github_workflow, "GitHub Actions workflow")
        
        # Check package.json scripts
        package_json = self.project_root / "package.json"
        if package_json.exists():
            try:
                with open(package_json) as f:
                    package_data = json.load(f)
                
                scripts = package_data.get("scripts", {})
                required_scripts = [
                    "test", "test:unit", "test:integration", "test:e2e",
                    "test:coverage", "test:performance", "test:all"
                ]
                
                for script in required_scripts:
                    if script in scripts:
                        print(f"✓ Script '{script}' defined")
                    else:
                        self.issues.append(f"✗ Missing script '{script}' in package.json")
                        
            except Exception as e:
                self.issues.append(f"✗ Could not parse package.json: {e}")
    
    def validate_test_data(self):
        """Validate test data and fixtures."""
        print("\n" + "="*50)
        print("VALIDATING TEST DATA")
        print("="*50)
        
        # Check fixture data files
        fixture_files = [
            ("tests/fixtures/data/tiktok/sample_videos.json", "TikTok sample data"),
            ("tests/fixtures/data/youtube/sample_channels.json", "YouTube sample data"),
            ("tests/fixtures/data/refined/expected_outputs.json", "Expected outputs"),
            ("tests/fixtures/mocks/api_responses.json", "Mock API responses")
        ]
        
        for file_path, description in fixture_files:
            if self.check_file_exists(self.project_root / file_path, description):
                # Validate JSON format
                try:
                    with open(self.project_root / file_path) as f:
                        json.load(f)
                    print(f"  ✓ Valid JSON format")
                except json.JSONDecodeError as e:
                    self.issues.append(f"✗ Invalid JSON in {file_path}: {e}")
    
    def run_validation(self):
        """Run complete validation."""
        print("SuData Test Configuration Validator")
        print("="*60)
        
        self.validate_python_environment()
        self.validate_node_environment()
        self.validate_test_structure()
        self.validate_service_configurations()
        self.validate_ci_configuration()
        self.validate_test_data()
        
        # Print summary
        print("\n" + "="*60)
        print("VALIDATION SUMMARY")
        print("="*60)
        
        if not self.issues and not self.warnings:
            print("✓ All validations passed! Test suite is properly configured.")
            return True
        
        if self.issues:
            print(f"\nISSUES FOUND ({len(self.issues)}):")
            for issue in self.issues:
                print(f"  {issue}")
        
        if self.warnings:
            print(f"\nWARNINGS ({len(self.warnings)}):")
            for warning in self.warnings:
                print(f"  {warning}")
        
        print(f"\nValidation {'FAILED' if self.issues else 'PASSED WITH WARNINGS'}")
        return len(self.issues) == 0

def main():
    """Main entry point."""
    project_root = Path(__file__).parent
    validator = TestConfigValidator(project_root)
    
    success = validator.run_validation()
    sys.exit(0 if success else 1)

if __name__ == "__main__":
    main()