baseline_scraper.py

#!/usr/bin/env python3
"""
Baseline article scraper for stylometric analysis.
Downloads articles from CEO Today and Green Matters to establish empirical baselines.
"""

import os
import re
import time
import json
import requests
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup
from typing import Dict, List, Optional
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)


class BaselineScraper:
    """Scraper for downloading baseline articles from news websites."""
    
    def __init__(self, output_dir: str = "baseline", delay: float = 1.0):
        """
        Initialize the scraper.
        
        Args:
            output_dir: Directory to save downloaded articles
            delay: Delay between requests to be respectful to servers
        """
        self.output_dir = output_dir
        self.delay = delay
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                           'AppleWebKit/537.36 (KHTML, like Gecko) '
                           'Chrome/120.0.0.0 Safari/537.36'),
            'Accept': ('text/html,application/xhtml+xml,application/xml;q=0.9,'
                        'image/webp,*/*;q=0.8'),
            'Accept-Language': 'en-US,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
        })
        
        # Create output directory
        os.makedirs(output_dir, exist_ok=True)
        
    def download_articles_from_site(self, site_config: Dict, num_articles: int = 15) -> Dict[str, str]:
        """
        Generic method to download articles from any configured site.
        
        Args:
            site_config: Dictionary with site configuration
                - name: Site name for logging and file prefixes
                - base_url: Main page URL
                - url_pattern: Pattern to match in article URLs
                - title_attr: Optional attribute name for article titles (e.g., 'title')
                - title_selector: Optional CSS selector for title extraction
            num_articles: Number of articles to download
            
        Returns:
            Dictionary mapping article titles to article content
        """
        site_name = site_config['name']
        base_url = site_config['base_url']
        url_pattern = site_config['url_pattern']
        title_attr = site_config.get('title_attr')  # e.g., 'title' attribute
        title_selector = site_config.get('title_selector')  # e.g., 'h2.article-title'
        link_selector = site_config.get('link_selector')  # e.g., 'a.news-item'
        exclude_patterns = site_config.get('exclude_patterns', [])  # patterns to exclude
        
        logger.info(f"Downloading {num_articles} articles from {site_name}...")
        
        articles = {}
        
        try:
            # Get the main page
            response = self.session.get(base_url)
            response.raise_for_status()
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Find article links
            article_links = []
            
            # Look for article links
            if link_selector:
                # Use specific selector for links (e.g., 'a.news-item')
                links = soup.select(link_selector)
            else:
                # Fallback to finding all links
                links = soup.find_all('a', href=True)
            
            for link in links:
                href = link.get('href', '')
                
                # Check if URL matches pattern and doesn't match exclude patterns
                if (url_pattern in href and 
                    href != base_url and
                    not any(exclude_pattern in href for exclude_pattern in exclude_patterns)):
                    
                    full_url = urljoin(base_url, href)
                    if full_url not in [link['url'] for link in article_links]:
                        # Extract title using site-specific method
                        title = self._extract_article_title(link, title_attr, title_selector)
                        
                        article_links.append({
                            'url': full_url,
                            'title': title
                        })
            
            logger.info(f"Found {len(article_links)} potential articles")
            
            # Download articles
            for i, article_info in enumerate(article_links[:num_articles]):
                try:
                    logger.info(f"Downloading article {i+1}/{num_articles}: {article_info['title'][:50]}...")
                    
                    article_content = self._download_article_content(article_info['url'], site_name)
                    if article_content:
                        # Clean title for filename
                        clean_title = re.sub(r'[^\w\s-]', '', article_info['title'])
                        clean_title = re.sub(r'\s+', '_', clean_title)[:50]
                        
                        articles[clean_title] = article_content
                        
                        # Save to file
                        filename = f"{site_name.lower().replace(' ', '_')}_{i+1:02d}_{clean_title}.txt"
                        filepath = os.path.join(self.output_dir, filename)
                        with open(filepath, 'w', encoding='utf-8') as f:
                            f.write(article_content)
                        
                        logger.info(f"Saved: {filename}")
                    
                    time.sleep(self.delay)  # Be respectful
                    
                except Exception as e:
                    logger.error(f"Failed to download article {i+1}: {e}")
                    continue
                    
        except Exception as e:
            logger.error(f"Failed to access {site_name}: {e}")
            
        logger.info(f"Successfully downloaded {len(articles)} {site_name} articles")
        return articles
    
    def _extract_article_title(self, link_element, title_attr: str = None, title_selector: str = None) -> str:
        """
        Extract article title using site-specific methods.
        
        Args:
            link_element: BeautifulSoup link element
            title_attr: Attribute name containing title (e.g., 'title')
            title_selector: CSS selector for title element
            
        Returns:
            Extracted title string
        """
        # Method 1: Use title attribute if specified
        if title_attr and link_element.get(title_attr):
            title = link_element.get(title_attr).strip()
            if title:
                return title
        
        # Method 2: Use CSS selector if specified
        if title_selector:
            title_elem = link_element.select_one(title_selector)
            if title_elem:
                title = title_elem.get_text(strip=True)
                if title:
                    return title
        
        # Method 3: Look for title in child elements (common patterns)
        title_selectors = ['h1', 'h2', 'h3', '.title', '.headline', '.article-title']
        for selector in title_selectors:
            title_elem = link_element.select_one(selector)
            if title_elem:
                title = title_elem.get_text(strip=True)
                if title:
                    return title
        
        # Method 4: Fallback to link text
        title = link_element.get_text(strip=True)
        if title:
            return title
        
        # Method 5: Final fallback
        return 'Untitled'
    
    def _download_article_content(self, url: str, site_name: str = None) -> Optional[str]:
        """Download content from a single article URL."""
        try:
            response = self.session.get(url)
            response.raise_for_status()
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Use site-specific extraction if available
            if site_name == 'Colossal':
                return self._extract_colossal_content(soup)
            
            # Extract article content
            content_parts = []
            
            # Look for article body in various possible locations
            content_selectors = [
                'article',
                '.entry-content',
                '.post-content',
                '.article-content',
                '.content',
                'main',
                '[data-testid="article-body"]'
            ]
            
            for selector in content_selectors:
                content_elem = soup.select_one(selector)
                if content_elem:
                    # Remove script and style elements
                    for script in content_elem(["script", "style"]):
                        script.decompose()
                    
                    # Get text content
                    text = content_elem.get_text(separator=' ', strip=True)
                    if len(text) > 200:  # Minimum content length
                        content_parts.append(text)
                        break
            
            # If no structured content found, try to extract from body
            if not content_parts:
                body = soup.find('body')
                if body:
                    # Remove navigation, footer, etc.
                    for elem in body.find_all(['nav', 'header', 'footer', 'aside']):
                        elem.decompose()
                    
                    text = body.get_text(separator=' ', strip=True)
                    if len(text) > 200:
                        content_parts.append(text)
            
            if content_parts:
                return content_parts[0]
            else:
                logger.warning(f"No content found for {url}")
                return None
                
        except Exception as e:
            logger.error(f"Error downloading {url}: {e}")
            return None
    
    def _extract_colossal_content(self, soup: BeautifulSoup) -> Optional[str]:
        """Extract clean article content from Colossal pages."""
        try:
            # For Colossal, we need to extract content from the main news page
            # since individual article pages are blocked
            
            # Look for article content in the main page structure
            text = soup.get_text(separator=' ', strip=True)
            
            # Try to find the article content by looking for specific patterns
            # Based on the downloaded content, we need to extract the actual article text
            
            # Look for content between "Colossal Home Page" and footer markers
            if 'Colossal Home Page' in text:
                parts = text.split('Colossal Home Page')
                if len(parts) > 1:
                    content = parts[1]
                    
                    # Remove footer content
                    footer_markers = [
                        "Let's make a better world together",
                        "Copyright ©",
                        "Terms of Use",
                        "Privacy Policy",
                        "For General Inquiries",
                        "For Press Inquiries"
                    ]
                    
                    for marker in footer_markers:
                        if marker in content:
                            content = content.split(marker)[0]
                    
                    # Clean up the content
                    content = re.sub(r'\s+', ' ', content).strip()
                    
                    # Remove navigation patterns
                    content = re.sub(r'\d{2}[a-z]?\s+[A-Z][a-z\s]+', '', content)
                    content = re.sub(r'\d{3}-\d{2}\s+[a-z\s]+', '', content)
                    content = re.sub(r'Page Sections for:.*?02[a-f]\s+[a-z\s]+', '', content)
                    
                    # Clean up multiple spaces again
                    content = re.sub(r'\s+', ' ', content).strip()
                    
                    if len(content) > 100:  # Ensure we have meaningful content
                        return content
            
            # Alternative approach: look for the main article text
            # The actual article content seems to be in the middle of the page
            if 'BY' in text:  # Look for author attribution
                parts = text.split('BY')
                if len(parts) > 1:
                    # Take content after the author line
                    content = parts[1]
                    
                    # Remove everything after "To Learn More" or similar
                    if 'To Learn More' in content:
                        content = content.split('To Learn More')[0]
                    if 'Return to Company Page' in content:
                        content = content.split('Return to Company Page')[0]
                    
                    # Clean up
                    content = re.sub(r'\s+', ' ', content).strip()
                    
                    if len(content) > 50:
                        return content
            
            return None
            
        except Exception as e:
            logger.error(f"Error extracting Colossal content: {e}")
            return None
    
    def download_ceo_today_articles(self, num_articles: int = 15) -> Dict[str, str]:
        """Download articles from CEO Today blog."""
        site_config = {
            'name': 'CEO_Today',
            'base_url': 'https://www.ceotodaymagazine.com/category/blog/',
            'url_pattern': '/2025/',  # CEO Today articles have year in URL
            'title_attr': 'title',    # CEO Today uses title attribute for article titles
            'exclude_patterns': ['/page/', '/category/']  # Exclude pagination and category links
        }
        return self.download_articles_from_site(site_config, num_articles)
    
    def download_green_matters_articles(self, num_articles: int = 15) -> Dict[str, str]:
        """Download articles from Green Matters news."""
        site_config = {
            'name': 'Green_Matters',
            'base_url': 'https://www.greenmatters.com/news',
            'url_pattern': '/news/'
        }
        return self.download_articles_from_site(site_config, num_articles)
    
    def download_colossal_articles(self, num_articles: int = 15) -> Dict[str, str]:
        """Download articles from Colossal news."""
        site_config = {
            'name': 'Colossal',
            'base_url': 'https://colossal.com/news/',
            'url_pattern': '',  # No specific pattern needed
            'link_selector': 'a.news-item'  # Look for links with class "news-item"
        }
        return self.download_articles_from_site(site_config, num_articles)
    
    
    def load_downloaded_articles(self) -> Dict[str, Dict[str, str]]:
        """
        Load previously downloaded articles from the baseline directory.
        
        Returns:
            Dictionary with publication names as keys, each containing
            article_name: article_content dictionaries
        """
        articles = {
            "CEO_Today": {},
            "Green_Matters": {},
            "Colossal": {}
        }
        
        if not os.path.exists(self.output_dir):
            logger.warning(f"Baseline directory {self.output_dir} does not exist")
            return articles
        
        for filename in os.listdir(self.output_dir):
            if filename.endswith('.txt'):
                filepath = os.path.join(self.output_dir, filename)
                
                try:
                    with open(filepath, 'r', encoding='utf-8') as f:
                        content = f.read().strip()
                    
                    if content:
                        # Extract publication and article name from filename
                        if filename.startswith('ceo_today_'):
                            pub = "CEO_Today"
                            article_name = filename[11:-4]  # Remove 'ceo_today_' and '.txt'
                        elif filename.startswith('green_matters_'):
                            pub = "Green_Matters"
                            article_name = filename[15:-4]  # Remove 'green_matters_' and '.txt'
                        elif filename.startswith('colossal_'):
                            pub = "Colossal"
                            article_name = filename[10:-4]  # Remove 'colossal_' and '.txt'
                        else:
                            continue
                        
                        articles[pub][article_name] = content
                        
                except Exception as e:
                    logger.error(f"Error loading {filename}: {e}")
        
        # Log summary
        for pub, pub_articles in articles.items():
            logger.info(f"Loaded {len(pub_articles)} articles from {pub}")
        
        return articles
    
    def download_all_baselines(self, articles_per_site: int = 15) -> Dict[str, Dict[str, str]]:
        """
        Download baseline articles from both sites.
        
        Args:
            articles_per_site: Number of articles to download from each site
            
        Returns:
            Dictionary with publication names as keys, each containing
            article_name: article_content dictionaries
        """
        logger.info("Starting baseline article download...")
        
        all_articles = {}
        
        # Download from CEO Today
        ceo_articles = self.download_ceo_today_articles(articles_per_site)
        if ceo_articles:
            all_articles["CEO_Today"] = ceo_articles
        
        # Download from Green Matters
        green_articles = self.download_green_matters_articles(articles_per_site)
        if green_articles:
            all_articles["Green_Matters"] = green_articles
        
        # Download from Colossal
        colossal_articles = self.download_colossal_articles(articles_per_site)
        if colossal_articles:
            all_articles["Colossal"] = colossal_articles
        
        # Save metadata
        metadata = {
            'download_date': time.strftime('%Y-%m-%d %H:%M:%S'),
            'articles_per_site': articles_per_site,
            'total_articles': sum(len(articles) for articles in all_articles.values()),
            'publications': list(all_articles.keys())
        }
        
        metadata_file = os.path.join(self.output_dir, 'metadata.json')
        with open(metadata_file, 'w') as f:
            json.dump(metadata, f, indent=2)
        
        logger.info(f"Download complete. Total articles: {metadata['total_articles']}")
        return all_articles


def main():
    """Main function for command-line usage."""
    import argparse
    
    parser = argparse.ArgumentParser(description='Download baseline articles for stylometric analysis')
    parser.add_argument('--output-dir', default='baseline', help='Output directory for articles')
    parser.add_argument('--articles-per-site', type=int, default=15, help='Articles to download per site')
    parser.add_argument('--delay', type=float, default=1.0, help='Delay between requests (seconds)')
    
    args = parser.parse_args()
    
    scraper = BaselineScraper(output_dir=args.output_dir, delay=args.delay)
    articles = scraper.download_all_baselines(args.articles_per_site)
    
    print(f"\nDownload Summary:")
    for pub, pub_articles in articles.items():
        print(f"  {pub}: {len(pub_articles)} articles")
    
    print(f"\nArticles saved to: {args.output_dir}/")
    print("You can now use these articles in your stylometric analysis.")


if __name__ == "__main__":
    main()