run_eda.py

"""
Main EDA Script
Orchestrates the entire EDA pipeline for the image tagging dataset.
"""

import sys
import json
from pathlib import Path
import pandas as pd
import logging

# Add eda package to path
sys.path.insert(0, str(Path(__file__).parent))

from eda.data_loader import DataLoader
from eda.schema_analyzer import SchemaAnalyzer
from eda.tag_analyzer import TagAnalyzer
from eda.image_analyzer import ImageAnalyzer
from eda.visualizer import Visualizer

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)


def save_summary_csv(df: pd.DataFrame, output_path: Path, name: str):
    """Save a DataFrame to CSV."""
    df.to_csv(output_path / f"{name}.csv", index=False, encoding='utf-8')
    logger.info(f"Saved {name} to {output_path / f'{name}.csv'}")


def save_summary_json(data: dict, output_path: Path, name: str):
    """Save a dictionary to JSON."""
    with open(output_path / f"{name}.json", 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=2)
    logger.info(f"Saved {name} to {output_path / f'{name}.json'}")


def main():
    """Main EDA pipeline."""
    # Configuration
    data_dir = "entities_dataset_v2"
    output_dir = Path("eda_output")
    output_dir.mkdir(exist_ok=True)
    
    logger.info("=" * 80)
    logger.info("Starting EDA Pipeline for Image Tagging Dataset")
    logger.info("=" * 80)
    
    # Step 1: Load data
    logger.info("\n[Step 1/7] Loading JSON files...")
    loader = DataLoader(data_dir)
    raw_data = loader.load_all_json_files()
    
    if not raw_data:
        logger.error("No data loaded! Exiting.")
        return
    
    logger.info(f"Loaded {len(raw_data)} records")
    if loader.failed_files:
        logger.warning(f"Failed to load {len(loader.failed_files)} files")
    
    # Step 2: Normalize to DataFrame
    logger.info("\n[Step 2/7] Normalizing data to DataFrame...")
    df = loader.normalize_to_dataframe()
    image_df = loader.get_image_level_dataframe()
    
    logger.info(f"Created normalized DataFrame with {len(df)} rows")
    logger.info(f"Created image-level DataFrame with {len(image_df)} images")
    
    # Step 3: Schema Analysis
    logger.info("\n[Step 3/7] Analyzing schema...")
    schema_analyzer = SchemaAnalyzer(raw_data)
    field_stats = schema_analyzer.discover_fields()
    schema_variability = schema_analyzer.analyze_schema_variability()
    missing_analysis = schema_analyzer.analyze_missing_values(df)
    nested_analysis = schema_analyzer.analyze_nested_structures()
    
    schema_report = schema_analyzer.generate_schema_report(df)
    print("\n" + schema_report)
    
    # Save schema analysis
    save_summary_json(field_stats, output_dir, "schema_field_stats")
    save_summary_json(schema_variability, output_dir, "schema_variability")
    save_summary_json(missing_analysis, output_dir, "missing_values")
    save_summary_json(nested_analysis, output_dir, "nested_structures")
    
    # Step 4: Tag Analysis
    logger.info("\n[Step 4/7] Analyzing tags and entities...")
    tag_analyzer = TagAnalyzer(df, image_df)
    unique_stats = tag_analyzer.analyze_unique_tags()
    tag_freq_df = tag_analyzer.analyze_tag_frequency()
    entity_name_freq_df = tag_analyzer.analyze_entity_name_frequency()
    long_tail_stats = tag_analyzer.analyze_long_tail(tag_freq_df)
    co_occurrence_df = tag_analyzer.analyze_co_occurrence(top_n=50)
    
    tag_report = tag_analyzer.generate_tag_report()
    print("\n" + tag_report)
    
    # Save tag analysis
    save_summary_csv(tag_freq_df, output_dir, "tag_frequencies")
    save_summary_csv(entity_name_freq_df, output_dir, "entity_name_frequencies")
    save_summary_csv(co_occurrence_df, output_dir, "tag_co_occurrences")
    save_summary_json(unique_stats, output_dir, "unique_tags_stats")
    save_summary_json(long_tail_stats, output_dir, "long_tail_stats")
    
    # Step 5: Image-Level Analysis
    logger.info("\n[Step 5/7] Analyzing image-level statistics...")
    image_analyzer = ImageAnalyzer(image_df)
    tags_per_image_stats = image_analyzer.analyze_tags_per_image()
    entity_types_stats = image_analyzer.analyze_entity_types_per_image()
    group_stats_df = image_analyzer.analyze_images_by_group()
    product_stats_df = image_analyzer.analyze_images_by_product()
    extreme_cases = image_analyzer.get_extreme_cases(n=10)
    
    image_report = image_analyzer.generate_image_report()
    print("\n" + image_report)
    
    # Save image analysis
    save_summary_csv(group_stats_df, output_dir, "group_statistics")
    save_summary_csv(product_stats_df, output_dir, "product_statistics")
    save_summary_csv(extreme_cases['zero_tags'], output_dir, "images_zero_tags")
    save_summary_csv(extreme_cases['many_tags'], output_dir, "images_many_tags")
    save_summary_json(tags_per_image_stats, output_dir, "tags_per_image_stats")
    save_summary_json(entity_types_stats, output_dir, "entity_types_stats")
    
    # Step 6: Visualizations
    logger.info("\n[Step 6/7] Creating visualizations...")
    visualizer = Visualizer(output_dir=str(output_dir))
    
    try:
        visualizer.plot_top_tags(tag_freq_df, top_n=30)
        visualizer.plot_tag_frequency_distribution(tag_freq_df, log_scale=True)
        visualizer.plot_tags_per_image(image_df)
        visualizer.plot_entity_name_frequency(entity_name_freq_df, top_n=20)
        if len(co_occurrence_df) > 0:
            visualizer.plot_co_occurrence_heatmap(co_occurrence_df, top_n=20)
        visualizer.plot_tags_by_group(group_stats_df, top_n=15)
        logger.info("All visualizations created successfully")
    except Exception as e:
        logger.error(f"Error creating visualizations: {e}", exc_info=True)
    
    # Step 7: Generate Summary Insights
    logger.info("\n[Step 7/7] Generating summary insights...")
    
    insights = {
        "dataset_overview": {
            "total_records": len(raw_data),
            "total_images": len(image_df),
            "total_tag_occurrences": len(df[df['entity_name'].notna()]),
            "unique_tags": unique_stats['unique_tag_combinations'],
            "unique_entity_types": unique_stats['unique_entity_names']
        },
        "data_quality": {
            "images_with_zero_tags": tags_per_image_stats['images_with_zero_tags'],
            "images_with_zero_tags_pct": (tags_per_image_stats['images_with_zero_tags'] / tags_per_image_stats['total_images']) * 100,
            "avg_tags_per_image": tags_per_image_stats['mean_tags_per_image'],
            "median_tags_per_image": tags_per_image_stats['median_tags_per_image']
        },
        "tag_distribution": {
            "very_rare_tags": long_tail_stats['very_rare_tags'],
            "rare_tags": long_tail_stats['rare_tags'],
            "common_tags": long_tail_stats['common_tags'],
            "long_tail_ratio": long_tail_stats['long_tail_ratio']
        },
        "recommendations": []
    }
    
    # Generate recommendations
    if long_tail_stats['long_tail_ratio'] > 0.5:
        insights["recommendations"].append(
            "High long-tail distribution detected. Consider filtering rare tags (<5 occurrences) for initial modeling."
        )
    
    if tags_per_image_stats['images_with_zero_tags'] / tags_per_image_stats['total_images'] > 0.1:
        insights["recommendations"].append(
            f"Significant portion ({insights['data_quality']['images_with_zero_tags_pct']:.1f}%) of images have zero tags. Review annotation quality."
        )
    
    if tags_per_image_stats['mean_tags_per_image'] < 3:
        insights["recommendations"].append(
            "Low average tags per image. This is a sparse multi-label problem."
        )
    
    save_summary_json(insights, output_dir, "summary_insights")
    
    # Print summary
    print("\n" + "=" * 80)
    print("SUMMARY INSIGHTS")
    print("=" * 80)
    print(f"Total Images: {insights['dataset_overview']['total_images']}")
    print(f"Unique Tags: {insights['dataset_overview']['unique_tags']}")
    print(f"Unique Entity Types: {insights['dataset_overview']['unique_entity_types']}")
    print(f"Average Tags per Image: {insights['data_quality']['avg_tags_per_image']:.2f}")
    print(f"Images with Zero Tags: {insights['data_quality']['images_with_zero_tags']} ({insights['data_quality']['images_with_zero_tags_pct']:.1f}%)")
    print(f"Long-tail Ratio: {insights['tag_distribution']['long_tail_ratio']:.2%}")
    print("\nRecommendations:")
    for rec in insights["recommendations"]:
        print(f"  - {rec}")
    print("=" * 80)
    
    logger.info("\nEDA Pipeline completed successfully!")
    logger.info(f"All outputs saved to: {output_dir.absolute()}")


if __name__ == "__main__":
    main()