stylometric_analyzer.py

import re
import os
import itertools
import random
import warnings
from typing import Dict, List, Tuple, Set, Optional
import numpy as np
import pandas as pd
import textwrap

from sentence_transformers import SentenceTransformer
import logging
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from scipy.optimize import linear_sum_assignment

import nltk
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

# Set random seeds for deterministic behavior
RANDOM_SEED = 42
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)

# Set torch seed for deterministic behavior in neural networks
try:
    import torch
    torch.manual_seed(RANDOM_SEED)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(RANDOM_SEED)
        torch.cuda.manual_seed_all(RANDOM_SEED)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False
except ImportError:
    pass

# Download required NLTK data if not already present
try:
    stopwords.words('english')
except LookupError:
    nltk.download('stopwords')
    nltk.download('punkt')

# Get English stop words
STOP_WORDS = set(stopwords.words('english'))

# Configure logging to silence sentence transformer and transformers messages
logging.getLogger("sentence_transformers").setLevel(logging.ERROR)
logging.getLogger("transformers").setLevel(logging.ERROR)
logging.getLogger("torch").setLevel(logging.ERROR)
logging.getLogger("huggingface_hub").setLevel(logging.ERROR)

# Suppress Kaleido logging (used by Plotly for image export)
logging.getLogger("kaleido").setLevel(logging.ERROR)
logging.getLogger("choreographer").setLevel(logging.ERROR)

# Suppress warnings
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=FutureWarning)

# Suppress tqdm warnings specifically
warnings.filterwarnings("ignore", message="IProgress not found.*")
warnings.filterwarnings("ignore", message=".*ipywidgets.*")

# stream logging
logging.basicConfig(
    level=logging.INFO, 
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

# Use disable_logging context manager to silence model loading messages
from sentence_transformers.util import disable_logging

with disable_logging(logging.CRITICAL):
    SEMANTIC_MODEL = SentenceTransformer('all-MiniLM-L6-v2')
    # Set seed for deterministic behavior in the model
    SEMANTIC_MODEL.seed = RANDOM_SEED
    # Test the model with a simple encoding to ensure it works
    test_embedding = SEMANTIC_MODEL.encode(
        "Test sentence", convert_to_tensor=True
    )
    assert test_embedding.shape[0] > 0, "Model encoding failed"
print("✓ Sentence transformer model loaded successfully")


class StylometricAnalyzer:
    """
    Enhanced Stylometric analysis tool with robust empirical baseline establishment.
    Requires minimum 10 articles per publication for statistical validity.
    """
    
    def __init__(self, model=SEMANTIC_MODEL, baseline_dir: str = "baseline", splitting_method: str = 'sentences'):
        self.baselines = {}
        self.tfidf_vectorizer = None
        self.model = model
        if splitting_method in ['sentences', 'paragraphs']:
            self.splitting_method = splitting_method
        else:
            raise ValueError("splitting_method must be 'sentences' or 'paragraphs'")
        self.fit_tfidf_baseline_from_dir(baseline_dir)

    def clean_text(self, text: str) -> str:
        """Clean and normalize text for analysis while
        preserving paragraph structure."""
        text = text.lower()
        # Preserve paragraph breaks by temporarily marking them
        text = re.sub(r'\n\s*\n', ' PARAGRAPH_BREAK ', text)
        text = re.sub(r'\n(?=\s)', ' PARAGRAPH_BREAK ', text)
        
        # Clean punctuation but preserve word boundaries
        text = re.sub(r'[^\w\s]', ' ', text)
        text = re.sub(r'\s+', ' ', text)
        
        # Restore paragraph breaks
        text = re.sub(r' PARAGRAPH_BREAK ', '\n\n', text)
        
        return text.strip()
    
    def clean_text_for_sentences(self, text: str) -> str:
        """Clean text while preserving sentence boundaries for sentence splitting."""
        text = text.lower()
        # Preserve sentence endings by temporarily marking them
        text = re.sub(r'([.!?])\s+', r' SENTENCE_END \1 ', text)
        text = re.sub(r'([.!?])$', r' SENTENCE_END \1', text)
        
        # Clean punctuation but preserve word boundaries
        text = re.sub(r'[^\w\s]', ' ', text)
        text = re.sub(r'\s+', ' ', text)
        
        # Restore sentence endings
        text = re.sub(r' SENTENCE_END ', '. ', text)
        
        return text.strip()
    
    def get_words(self, text: str, filter_stopwords: bool = True) -> List[str]:
        """Extract words from text, optionally filtering out stop words."""
        cleaned = self.clean_text(text)
        words = [word for word in cleaned.split() if len(word) > 0]
        
        if filter_stopwords and STOP_WORDS:
            # Filter out stop words
            words = [word for word in words if word.lower() not in STOP_WORDS]
        
        return words
    
    def calculate_chunk_similarity_hungarian(self, chunks1: List[str], chunks2: List[str], 
                                              similarity_func, top_k: int = 3) -> Dict:
        """
        Use Hungarian algorithm to find optimal chunk-to-chunk matching.
        
        Args:
            chunks1: List of chunks from first text
            chunks2: List of chunks from second text  
            similarity_func: Function that takes two chunks and returns similarity score (0-1)
            top_k: Number of top similar chunk pairs to return
        Returns:
            Dictionary with optimal matching results
        """
        if not chunks1 or not chunks2:
            return {
                'hungarian_avg_similarity': 0.0,
                'matched_pairs': [],
                'matched_similarities': [],
                'n_matches': 0
            }
        
        # Create cost matrix (1 - similarity) since Hungarian minimizes cost
        cost_matrix = np.zeros((len(chunks1), len(chunks2)))
        
        for i, chunk1 in enumerate(chunks1):
            for j, chunk2 in enumerate(chunks2):
                similarity = similarity_func(chunk1, chunk2)
                cost_matrix[i, j] = 1 - similarity
        
        # Handle rectangular matrices
        if len(chunks1) != len(chunks2):
            max_len = max(len(chunks1), len(chunks2))
            padded_matrix = np.full((max_len, max_len), 1.0)
            padded_matrix[:len(chunks1), :len(chunks2)] = cost_matrix
        else:
            padded_matrix = cost_matrix
        
        # Apply Hungarian algorithm
        row_indices, col_indices = linear_sum_assignment(padded_matrix)
        
        # Calculate results for matched pairs
        matched_similarities = []
        matched_pairs = []
        for row, col in zip(row_indices, col_indices):
            if row < len(chunks1) and col < len(chunks2):
                similarity = 1 - padded_matrix[row, col]
                matched_similarities.append(similarity)
                matched_pairs.append((row, col))
        # provide the top k pairs by similarity
        # Create list of (pair, similarity) tuples and sort by similarity
        pair_similarity_tuples = list(zip(matched_pairs, matched_similarities))
        sorted_pairs = sorted(pair_similarity_tuples, key=lambda x: x[1], reverse=True)
        
        top_k_pairs = [pair for pair, _ in sorted_pairs[:top_k]]
        top_k_similarities = [sim for _, sim in sorted_pairs[:top_k]]
        top_k_matched_chunks = [(chunks1[i], chunks2[j]) for i, j in top_k_pairs]
        return {
            'hungarian_avg_similarity': np.mean(matched_similarities) if matched_similarities else 0.0,
            'hungarian_median_similarity': np.median(matched_similarities) if matched_similarities else 0.0,
            'hungarian_max_similarity': np.max(matched_similarities) if matched_similarities else 0.0,
            'matched_pairs': matched_pairs,
            'matched_similarities': matched_similarities,
            'n_matches': len(matched_pairs),
            'top_k_matched_chunks': top_k_matched_chunks,
            'top_k_similarities': top_k_similarities
        }

    def semantic_similarity_func(self, chunk1: str, chunk2: str) -> float:
        """Calculate semantic similarity between two chunks."""
        
        embeddings = self.model.encode([chunk1, chunk2])
        similarity = cosine_similarity(embeddings[0:1], embeddings[1:2])[0][0]
        return float(similarity)
    
    def tfidf_similarity_func(self, chunk1: str, chunk2: str) -> float:
        """Calculate TF-IDF similarity between two chunks using fitted vectorizer."""
        try:
            if not self.tfidf_vectorizer:
                logger.warning("TF-IDF vectorizer not fitted. Skipping TF-IDF similarity.")
                return 0.0
            
            vectors = self.tfidf_vectorizer.transform([chunk1, chunk2])
            similarity = cosine_similarity(vectors[0], vectors[1])[0][0]
            return float(similarity)
        except Exception as e:
            logger.warning(f"TF-IDF similarity calculation failed: {e}")
            return 0.0
    
    def bleu_similarity_func(self, chunk1: str, chunk2: str) -> float:
        """Calculate BLEU similarity between two chunks."""
        result = self.calculate_bleu_score(chunk1, chunk2)
        return result.get('bleu_score', 0.0)
    
    def jaccard_similarity_func(self, chunk1: str, chunk2: str) -> float:
        """Calculate Jaccard similarity between two chunks."""
        words1 = set(self.get_words(chunk1, filter_stopwords=True))
        words2 = set(self.get_words(chunk2, filter_stopwords=True))
        return self.calculate_jaccard_similarity(words1, words2)
    
    def calculate_jaccard_similarity(self, set1: Set, set2: Set) -> float:
        """Calculate Jaccard similarity coefficient between two sets."""
        intersection = set1.intersection(set2)
        union = set1.union(set2)
        return len(intersection) / len(union) if len(union) > 0 else 0.0
    
    def calculate_bleu_score(self, text1: str, text2: str, 
                           smoothing: bool = True) -> Dict[str, float]:
        """
        Calculate BLEU scores between two texts.
        
        Args:
            text1: Reference text
            text2: Candidate text
            smoothing: Whether to use smoothing for short texts
            
        Returns:
            Dictionary with BLEU scores for different n-gram orders
        """
  
        try:
            # Tokenize texts
            reference = word_tokenize(text1.lower())
            candidate = word_tokenize(text2.lower())
            
            # Skip if either text is too short
            if len(reference) < 2 or len(candidate) < 2:
                return {
                    'bleu_score': 0.0,
                    'error': 'Texts too short for BLEU'
                }
            
            # Use smoothing function for short texts
            smoothing_fn = SmoothingFunction().method1 if smoothing else None
            
            # Calculate BLEU score with standard weights (equivalent to the previous average)
            bleu_score = sentence_bleu([reference], candidate, 
                                     smoothing_function=smoothing_fn)
            
            return {
                'bleu_score': bleu_score
            }
            
        except Exception as e:
            return {
                'bleu_score': 0.0,
                'error': str(e)
            }
    
    def split_into_sentences(self, text: str, min_words: int = 5) -> List[str]:
        """
        Split text into sentences and filter out too short ones.
        
        Args:
            text: Input text to split
            min_words: Minimum number of words required for a sentence
            
        Returns:
            List of valid sentences
        """
        # Clean the text while preserving sentence structure
        cleaned_text = self.clean_text_for_sentences(text)
        
        # Split by sentence endings (., !, ?) followed by whitespace
        sentences = re.split(r'[.!?]+\s+', cleaned_text.strip())
        
        # Filter sentences
        valid_sentences = []
        for sentence in sentences:
            sentence = sentence.strip()
            if sentence and len(self.get_words(sentence)) >= min_words:
                valid_sentences.append(sentence)
        
        return valid_sentences
    
    def split_into_paragraphs(self, text: str, min_words: int = 10) -> List[str]:
        """
        Split text into paragraphs and filter out too short ones.
        
        Args:
            text: Input text to split
            min_words: Minimum number of words required for a paragraph
            
        Returns:
            List of valid paragraphs
        """
        # Clean the text while preserving paragraph structure
        cleaned_text = self.clean_text(text)
        
        # Split by double newlines (paragraph breaks)
        paragraphs = re.split(r'\n\s*\n', cleaned_text.strip())
        
        # Filter paragraphs
        valid_paragraphs = []
        for para in paragraphs:
            para = para.strip()
            if para and len(self.get_words(para)) >= min_words:
                valid_paragraphs.append(para)
        
        return valid_paragraphs

    def fit_tfidf_baseline_from_dir(self, baseline_dir: str = "baseline",
                                     ngram_range: Tuple[int, int] = (1, 2),
                                     max_features: Optional[int] = None,
                                     min_df: int = 2,
                                     max_df: float = 0.9) -> None:
        """
        Fit a TF-IDF vectorizer on a baseline corpus located in a directory.

        Args:
            baseline_dir: Directory containing baseline .txt documents
            ngram_range: n-gram range for TF-IDF
            max_features: Optional cap on vocabulary size
            min_df: Ignore terms with document frequency lower than this
            max_df: Ignore terms with document frequency higher than this fraction
        """
        if not os.path.isdir(baseline_dir):
            print(f"Warning: Baseline directory not found: {baseline_dir}")
            return

        corpus_texts = []
        for filename in os.listdir(baseline_dir):
            if not filename.lower().endswith('.txt'):
                continue
            file_path = os.path.join(baseline_dir, filename)
            try:
                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                    text = f.read()
                    # Use cleaner to normalize; TF-IDF will tokenize itself
                    corpus_texts.append(self.clean_text(text))
            except Exception:
                continue

        if not corpus_texts:
            print("Warning: No baseline texts found for TF-IDF fitting.")
            return

        vectorizer = TfidfVectorizer(
            ngram_range=ngram_range,
            lowercase=True,
            max_features=max_features,
            min_df=min_df,
            max_df=max_df,
            stop_words='english'
        )
        vectorizer.fit(corpus_texts)
        self.tfidf_vectorizer = vectorizer
        print("✓ TF-IDF vectorizer fitted from baseline directory")

    def compare_texts(self, text1: str, text2: str,
                      min_chunk_words: int = 5,
                      top_k: int = 3) -> Dict:
        """Compare two texts across multiple similarity metrics.
        
        Args:
            text1: First text to compare
            text2: Second text to compare
            min_chunk_words: Minimum number of words required for a chunk
            filter_stopwords: Whether to filter out stop words
            top_k: Number of top similar chunks to return
        """
        
        # Split texts into chunks based on method
        if self.splitting_method == 'sentences':
            chunks1 = self.split_into_sentences(text1, min_chunk_words)
            chunks2 = self.split_into_sentences(text2, min_chunk_words)
        elif self.splitting_method == 'paragraphs':
            chunks1 = self.split_into_paragraphs(text1, min_chunk_words)
            chunks2 = self.split_into_paragraphs(text2, min_chunk_words)
        else:
            raise ValueError("splitting_method must be 'sentences' or 'paragraphs'")

        # Calculate semantic similarity using Hungarian algorithm
        semantic_sim_result = self.calculate_chunk_similarity_hungarian(
            chunks1, chunks2, self.semantic_similarity_func, top_k=top_k
        )
        semantic_sim_result = {
            'semantic_avg_score': semantic_sim_result['hungarian_avg_similarity'],
            'semantic_median_score': semantic_sim_result['hungarian_median_similarity'],
            'semantic_max_score': semantic_sim_result['hungarian_max_similarity'],
            'semantic_matched_pairs': semantic_sim_result['matched_pairs'],
            'semantic_n_matches': semantic_sim_result['n_matches'],
            'semantic_top_k_matched_chunks': semantic_sim_result['top_k_matched_chunks'],
            'semantic_top_k_similarities': semantic_sim_result['top_k_similarities']
        }
        
        # Calculate TF-IDF similarity using Hungarian algorithm
        tfidf_sim_result = self.calculate_chunk_similarity_hungarian(
            chunks1, chunks2, self.tfidf_similarity_func, top_k=top_k
        )
        tfidf_sim_result = {
            'tfidf_avg_score': tfidf_sim_result['hungarian_avg_similarity'],
            'tfidf_median_score': tfidf_sim_result['hungarian_median_similarity'],
            'tfidf_max_score': tfidf_sim_result['hungarian_max_similarity'],
            'tfidf_matched_pairs': tfidf_sim_result['matched_pairs'],
            'tfidf_n_matches': tfidf_sim_result['n_matches'],
            'tfidf_top_k_matched_chunks': tfidf_sim_result['top_k_matched_chunks'],
            'tfidf_top_k_similarities': tfidf_sim_result['top_k_similarities']
        }
        
        # Calculate BLEU similarity using Hungarian algorithm
        bleu_sim_result = self.calculate_chunk_similarity_hungarian(
            chunks1, chunks2, self.bleu_similarity_func
        )
        bleu_sim_result = {
            'bleu_avg_score': bleu_sim_result['hungarian_avg_similarity'],
            'bleu_median_score': bleu_sim_result['hungarian_median_similarity'],
            'bleu_max_score': bleu_sim_result['hungarian_max_similarity'],
            'bleu_matched_pairs': bleu_sim_result['matched_pairs'],
            'bleu_n_matches': bleu_sim_result['n_matches'],
            'bleu_top_k_matched_chunks': bleu_sim_result['top_k_matched_chunks'],
            'bleu_top_k_similarities': bleu_sim_result['top_k_similarities']
        }
        
        # Calculate Jaccard similarity using Hungarian algorithm
        jaccard_sim_result = self.calculate_chunk_similarity_hungarian(
            chunks1, chunks2, self.jaccard_similarity_func
        )
        jaccard_sim_result = {
            'jaccard_avg_score': jaccard_sim_result['hungarian_avg_similarity'],
            'jaccard_median_score': jaccard_sim_result['hungarian_median_similarity'],
            'jaccard_max_score': jaccard_sim_result['hungarian_max_similarity'],
            'jaccard_matched_pairs': jaccard_sim_result['matched_pairs'],
            'jaccard_n_matches': jaccard_sim_result['n_matches'],
            'jaccard_top_k_matched_chunks': jaccard_sim_result['top_k_matched_chunks'],
            'jaccard_top_k_similarities': jaccard_sim_result['top_k_similarities']
        }

        return {
            'chunks1': chunks1,
            'chunks2': chunks2,
            **semantic_sim_result,
            **tfidf_sim_result,
            **bleu_sim_result,
            **jaccard_sim_result,
        }

    def establish_baselines(self, baseline_articles: Dict[str, Dict[str, str]], 
                                  sample_size: int = None) -> Dict:
        """
        Establish robust empirical baselines from cross-publication comparisons.
        
        Args:
            baseline_articles: Articles by publication for baseline calculation
            max_combinations: Maximum combinations to sample per publication pair
            
        Returns:
            Dictionary with baseline statistics for all metrics
        """
        print("=== ESTABLISHING BASELINES ===")
 
        publications = list(baseline_articles.keys())
        # Generate all publication pairs
        publication_pairs = list(itertools.combinations(publications, 2))
        print(f"Found {len(publication_pairs)} publication pairs")
        
        # Collect baseline comparisons
        baseline_results = []
        total_comparisons = 0
        
        for pub1, pub2 in publication_pairs:
            articles1 = list(baseline_articles[pub1].values())
            articles2 = list(baseline_articles[pub2].values())
            
            # Sample combinations if max_combinations specified
            if sample_size:
                n1 = min(len(articles1), sample_size)
                n2 = min(len(articles2), sample_size)
                articles1 = random.sample(articles1, n1)
                articles2 = random.sample(articles2, n2)
            
            print(f"Comparing {len(articles1)} vs {len(articles2)} articles")
            # Compare all pairs between these publications
            for article1 in articles1:
                for article2 in articles2:
                    comparison = self.compare_texts(article1, article2)
                    baseline_results.append(comparison)
                    total_comparisons += 1
        
        print(f"Generated {total_comparisons} baseline comparisons")
        
        # Extract metric values
        jaccard_values = [r['jaccard_avg_score'] for r in baseline_results]
        jaccard_median_values = [r.get('jaccard_median_score', 0.0) for r in baseline_results]
        jaccard_max_values = [r.get('jaccard_max_score', 0.0) for r in baseline_results]
        
        semantic_avg_values = [r['semantic_avg_score'] for r in baseline_results]
        semantic_max_values = [r.get('semantic_max_score', 0.0) for r in baseline_results]
        semantic_median_values = [r.get('semantic_median_score', 0.0) for r in baseline_results]
        
        tfidf_values = [r['tfidf_avg_score'] for r in baseline_results]
        tfidf_median_values = [r.get('tfidf_median_score', 0.0) for r in baseline_results]
        tfidf_max_values = [r.get('tfidf_max_score', 0.0) for r in baseline_results]
        
        bleu_values = [r['bleu_avg_score'] for r in baseline_results]
        bleu_median_values = [r.get('bleu_median_score', 0.0) for r in baseline_results]
        bleu_max_values = [r.get('bleu_max_score', 0.0) for r in baseline_results]
        
        # Calculate baseline statistics
        baselines = {
            # Central tendencies
            'expected_jaccard_avg': np.mean(jaccard_values),
            'expected_jaccard_median': np.mean(jaccard_median_values),
            'expected_jaccard_max': np.mean(jaccard_max_values),
            
            'expected_semantic_avg': np.mean(semantic_avg_values),
            'expected_semantic_max': np.mean(semantic_max_values),
            'expected_semantic_median': np.mean(semantic_median_values),
            
            'expected_tfidf_avg': np.mean(tfidf_values),
            'expected_tfidf_median': np.mean(tfidf_median_values),
            'expected_tfidf_max': np.mean(tfidf_max_values),
            
            'expected_bleu_avg': np.mean(bleu_values),
            'expected_bleu_median': np.mean(bleu_median_values),
            'expected_bleu_max': np.mean(bleu_max_values),
            
            # Variability measures
            'std_jaccard_avg': np.std(jaccard_values),
            'std_jaccard_median': np.std(jaccard_median_values),
            'std_jaccard_max': np.std(jaccard_max_values),
            
            'std_semantic_avg': np.std(semantic_avg_values),
            'std_semantic_median': np.std(semantic_median_values),
            'std_semantic_max': np.std(semantic_max_values),
            
            'std_tfidf_avg': np.std(tfidf_values),
            'std_tfidf_median': np.std(tfidf_median_values),
            'std_tfidf_max': np.std(tfidf_max_values),
            
            'std_bleu_avg': np.std(bleu_values),
            'std_bleu_median': np.std(bleu_median_values),
            'std_bleu_max': np.std(bleu_max_values),
            
            # Metadata
            'n_comparisons': total_comparisons,
            'n_publications': len(publications),
            'n_publication_pairs': len(publication_pairs),
            'raw_data': baseline_results
        }
        
        # Print baseline summary
        print(f"\nBaseline Summary:")
        print(f"  Jaccard avg: {baselines['expected_jaccard_avg']:.4f} ± {baselines['std_jaccard_avg']:.4f}")
        print(f"  Semantic avg: {baselines['expected_semantic_avg']:.3f} ± {baselines['std_semantic_avg']:.3f}")
        print(f"  TF-IDF avg: {baselines['expected_tfidf_avg']:.3f} ± {baselines['std_tfidf_avg']:.3f}")
        print(f"  BLEU avg: {baselines['expected_bleu_avg']:.3f} ± {baselines['std_bleu_avg']:.3f}")
        
        return baselines
    
    def generate_baseline_distribution_report(self, baselines: Dict) -> str:
        """Generate a detailed statistical report of the baseline distribution."""
        report = []
        report.append("=== BASELINE DISTRIBUTION ANALYSIS ===")
        report.append(f"Sample size: {baselines['n_comparisons']} comparisons")
        report.append(f"Publications: {baselines['n_publications']}")
        report.append(f"Publication pairs: {baselines['n_publication_pairs']}")
        report.append("Method: Cross-publication comparisons (independent publications)")
        report.append("")
        
        # Jaccard distribution (chunk-level)
        report.append("Jaccard Similarity (Chunk-level) Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_jaccard_avg']:.4f} ± {baselines['std_jaccard_avg']:.4f}")
        report.append("")
        
        # Semantic similarity distributions
        report.append("Semantic Similarity (Average) Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_semantic_avg']:.3f} ± {baselines['std_semantic_avg']:.3f}")
        report.append("")
        
        report.append("Semantic Similarity (Maximum) Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_semantic_max']:.3f} ± {baselines['std_semantic_max']:.3f}")
        report.append("")
        
        report.append("Semantic Similarity (Median) Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_semantic_median']:.3f} ± {baselines['std_semantic_median']:.3f}")
        report.append("")
        
        # TF-IDF similarity distribution
        report.append("TF-IDF Similarity Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_tfidf_avg']:.3f} ± {baselines['std_tfidf_avg']:.3f}")
        report.append("")
        
        # BLEU score distribution
        report.append("BLEU Score Distribution:")
        report.append(f"  Mean ± SD: {baselines['expected_bleu_avg']:.3f} ± {baselines['std_bleu_avg']:.3f}")
        
        return "\n".join(report)
    
    def pairwise_analysis(
        self,
        articles: Dict[str, str],
        top_k: int = 3
    ) -> pd.DataFrame:
        """Perform pairwise analysis with enhanced statistical assessment."""
        print(f"\n=== PAIRWISE ANALYSIS ===")
        print(f"Analyzing {len(articles)} articles ({len(articles)*(len(articles)-1)//2} pairs)")
        
        results = []
        article_names = list(articles.keys())
        
        for i in range(len(article_names)):
            for j in range(i + 1, len(article_names)):
                name1, name2 = article_names[i], article_names[j]
                comparison = self.compare_texts(
                    articles[name1], articles[name2],
                    top_k=top_k
                )
                
                result = {
                    'pair': f"{name1} vs {name2}",
                    'article1': name1,
                    'article2': name2,
                    **comparison
                }

                results.append(result)

                print(f"{name1} vs {name2} done")
        
        return pd.DataFrame(results)
    
    def generate_results_overview(
        self,
        results_df: pd.DataFrame,
        overview_cols: List[str] = [
            'jaccard_avg_score', 'semantic_avg_score', 'tfidf_avg_score', 'bleu_avg_score'
        ]
    ) -> str:
        """
        Generate a comprehensive overview of pairwise analysis results.
        
        Args:
            results_df: DataFrame with pairwise comparison results
            
        Returns:
            Formatted string with results overview
        """
        overview = []
        overview.append("=== RESULTS OVERVIEW ===")
        
        # Key columns for overview (including semantic similarity)
        
        available_cols = [col for col in overview_cols if col in results_df.columns]

        overview.append(f"Total pairwise comparisons: {len(results_df)}")
        overview.append("\nSimilarity ranges:")
        for col in available_cols:
            overview.append(f"  {col.replace('_score', '').title()}: {results_df[col].min():.3f} - {results_df[col].max():.3f}")
        overview.append("-"*100)
        
        return "\n".join(overview)

    def top_k_matched_chunks_report(
        self,
        results_df: pd.DataFrame,
        metric: str = 'semantic',  # 'semantic', 'tfidf', 'bleu', 'jaccard'
        max_width: int = 80
    ) -> str:
        """Generate a report of the top k matched chunks."""
        report = []
        report.append("=== TOP K MATCHED CHUNKS ===")
        top_k_matched_chunks_key = f'{metric}_top_k_matched_chunks'
        top_k_similarities_key = f'{metric}_top_k_similarities'  # jaccard_top_k_matched_chunks
        
        # Iterate through all rows to show top k matches for each pair
        for idx, row in results_df.iterrows():
            pair_name = row['pair']
            matches = row[top_k_matched_chunks_key]
            similarities = row[top_k_similarities_key]
            
            report.append(f"\n{pair_name}:")
            report.append(f"Top {len(matches)} matched chunks: {metric.replace('_score', '').title()}")
            
            for k in range(len(matches)):
                report.append(f"similarity: {similarities[k]:.3f}")
                report.append(f"\n  Text 1: {textwrap.fill(matches[k][0], width=max_width, initial_indent='', subsequent_indent='    ')}")
                report.append(f"\n  Text 2: {textwrap.fill(matches[k][1], width=max_width, initial_indent='', subsequent_indent='    ')}")
                report.append("")
        return "\n".join(report)
    
    def statistical_analysis(
        self,
        results_df: pd.DataFrame,
        baselines: Dict,
        metric: str = 'semantic'
    ) -> pd.DataFrame:
        """Compare results with baseline and report statistical significance."""
        # Get baseline statistics for the metric from the baselines dict
        baseline_mean = baselines[f'expected_{metric}_avg']
        baseline_std = baselines[f'std_{metric}_avg']
        
        # Create a copy of results_df to add statistical analysis
        analysis_df = results_df.copy()
        
        # Calculate z-scores for each pair
        z_scores = []
        significance_levels = []
        
        for _, row in results_df.iterrows():
            pair_score = row[f'{metric}_avg_score']
            z_score = (pair_score - baseline_mean) / baseline_std
            z_scores.append(z_score)
            
            # Determine significance level
            if abs(z_score) >= 2.58:
                significance = "Highly Significant (p < 0.01)"
            elif abs(z_score) >= 1.96:
                significance = "Significant (p < 0.05)"
            elif abs(z_score) >= 1.645:
                significance = "Marginally Significant (p < 0.10)"
            else:
                significance = "Not Significant"
            
            significance_levels.append(significance)
        
        # Add columns to analysis_df
        analysis_df[f'{metric}_z_score'] = z_scores
        analysis_df[f'{metric}_significance'] = significance_levels
        analysis_df[f'{metric}_baseline_mean'] = baseline_mean
        analysis_df[f'{metric}_baseline_std'] = baseline_std
        
        # generate a report of the results
        report = []
        report.append("=== STATISTICAL ANALYSIS ===")
        report.append(f"Metric: {metric}")
        report.append(f"Baseline mean: {baseline_mean:.4f}")
        report.append(f"Baseline std: {baseline_std:.4f}")
        report.append("")
        
        for idx, row in analysis_df.iterrows():
            pair_name = row['pair']
            observed_value = row[f'{metric}_avg_score']
            z_score = row[f'{metric}_z_score']
            significance = row[f'{metric}_significance']
            
            # Extract article titles from pair name (assuming format like "title1 vs title2")
            if " vs " in pair_name:
                article1_title, article2_title = pair_name.split(" vs ", 1)
            else:
                article1_title = pair_name
                article2_title = "Unknown"
            
            report.append(f"Pair {idx + 1}:")
            report.append(f"  Article 1: {article1_title}")
            report.append(f"  Article 2: {article2_title}")
            report.append(f"  Expected: {baseline_mean:.4f}")
            report.append(f"  Observed: {observed_value:.4f}")
            report.append(f"  Z-score: {z_score:.3f}")
            report.append(f"  Significance: {significance}")
            report.append("")
        
        return "\n".join(report), analysis_df