utils.py

"""Utility functions for SeLoger scraper."""

import re
import logging
from typing import Optional, List, Tuple
from urllib.parse import urljoin, urlparse
from bs4 import BeautifulSoup

from config import (
    BASE_URL,
    PROPERTY_TYPE_TRANSLATIONS,
    CONTRACT_TYPE_TRANSLATIONS,
)

logger = logging.getLogger(__name__)


def extract_listing_id(url: str) -> Optional[str]:
    """Extract listing ID from property URL.

    Args:
        url: Property detail URL

    Returns:
        Listing ID or None
    """
    # URL format: /annonces/achat/appartement/paris-18eme-75/montmartre/256698393.htm
    match = re.search(r"/(\d+)\.htm", url)
    if match:
        return match.group(1)
    return None


def extract_contract_type(url: str) -> Optional[str]:
    """Extract contract type from URL.

    Args:
        url: Property URL

    Returns:
        Contract type (Buy/Rent) or None
    """
    # URL format: /annonces/achat/... or /annonces/location/...
    if "/achat/" in url or "/vente/" in url:
        return "Buy"
    elif "/location/" in url:
        return "Rent"
    return None


def parse_price(price_text: str) -> Optional[int]:
    """Parse price from text.

    Args:
        price_text: Price text like "310 000 €" or "310000 €"

    Returns:
        Price as integer or None
    """
    if not price_text:
        return None
    # Remove spaces, currency symbols, and non-numeric chars except digits
    cleaned = re.sub(r"[^\d]", "", price_text)
    if cleaned:
        return int(cleaned)
    return None


def parse_price_per_sqm(text: str) -> Optional[float]:
    """Parse price per square meter.

    Args:
        text: Text containing price per sqm like "10 690 €/m²"

    Returns:
        Price per sqm as float or None
    """
    if not text:
        return None
    match = re.search(r"([\d\s]+)\s*€/m²", text)
    if match:
        cleaned = re.sub(r"\s", "", match.group(1))
        if cleaned:
            return float(cleaned)
    return None


def parse_area(text: str) -> Optional[float]:
    """Parse living area from text.

    Args:
        text: Text containing area like "29 m²" or "85,9 m²"

    Returns:
        Area as float or None
    """
    if not text:
        return None
    match = re.search(r"([\d,\.]+)\s*m²", text)
    if match:
        area_str = match.group(1).replace(",", ".")
        return float(area_str)
    return None


def parse_rooms(text: str) -> Optional[int]:
    """Parse number of rooms.

    Args:
        text: Text containing rooms like "2 pièces" or "1 pièce"

    Returns:
        Number of rooms or None
    """
    if not text:
        return None
    match = re.search(r"(\d+)\s*pièces?", text)
    if match:
        return int(match.group(1))
    return None


def parse_bedrooms(text: str) -> Optional[int]:
    """Parse number of bedrooms.

    Args:
        text: Text containing bedrooms like "1 chambre" or "2 chambres"

    Returns:
        Number of bedrooms or None
    """
    if not text:
        return None
    match = re.search(r"(\d+)\s*chambres?", text)
    if match:
        return int(match.group(1))
    return None


def parse_floor(text: str) -> Tuple[Optional[str], Optional[int]]:
    """Parse floor information.

    Args:
        text: Text like "RDC/3" or "Étage 3/6" or "Rez-de-chaussée/3 étages"

    Returns:
        Tuple of (floor, total_floors)
    """
    if not text:
        return None, None

    floor = None
    total_floors = None

    # Handle "RDC/3" or "Étage 3/6"
    if "RDC" in text or "Rez-de-chaussée" in text.lower():
        floor = "Ground floor"
        match = re.search(r"/(\d+)", text)
        if match:
            total_floors = int(match.group(1))
    else:
        match = re.search(r"[Éé]tage\s*(\d+)/(\d+)", text)
        if match:
            floor = f"Floor {match.group(1)}"
            total_floors = int(match.group(2))
        else:
            match = re.search(r"(\d+)/(\d+)", text)
            if match:
                floor = f"Floor {match.group(1)}"
                total_floors = int(match.group(2))

    return floor, total_floors


def parse_postal_code(text: str) -> Optional[str]:
    """Extract postal code from location text.

    Args:
        text: Location text like "Paris 18ème (75018)"

    Returns:
        Postal code or None
    """
    if not text:
        return None
    match = re.search(r"\((\d{5})\)", text)
    if match:
        return match.group(1)
    return None


def parse_city(text: str) -> Optional[str]:
    """Extract city name from location text.

    Args:
        text: Location text like "Montmartre, Paris 18ème (75018)"

    Returns:
        City name or None
    """
    if not text:
        return None
    # Remove postal code
    text = re.sub(r"\s*\(\d{5}\)", "", text)
    # Split by comma and get the last part (city)
    parts = text.split(",")
    if len(parts) >= 2:
        return parts[-1].strip()
    return text.strip()


def parse_district(text: str) -> Optional[str]:
    """Extract district/neighborhood from location text.

    Args:
        text: Location text like "Montmartre, Paris 18ème (75018)"

    Returns:
        District name or None
    """
    if not text:
        return None
    # Split by comma
    parts = text.split(",")
    if len(parts) >= 2:
        return parts[0].strip()
    return None


def parse_property_type(text: str) -> Optional[str]:
    """Parse property type from text.

    Args:
        text: Property type text like "Appartement à vendre"

    Returns:
        Normalized property type or None
    """
    if not text:
        return None

    text_lower = text.lower()
    for french, english in PROPERTY_TYPE_TRANSLATIONS.items():
        if french.lower() in text_lower:
            return english

    # Return first word capitalized if no match
    return text.split()[0].capitalize() if text else None


def parse_total_count(html: str) -> int:
    """Parse total property count from listing page.

    Args:
        html: HTML content of listing page

    Returns:
        Total count of properties
    """
    soup = BeautifulSoup(html, "lxml")

    # Look for h1 with count like "20 180 maisons et appartements à vendre"
    h1 = soup.find("h1")
    if h1:
        text = h1.get_text()
        # Extract number from beginning
        match = re.search(r"^([\d\s]+)", text)
        if match:
            count_str = re.sub(r"\s", "", match.group(1))
            if count_str:
                return int(count_str)

    return 0


def parse_property_urls(html: str) -> List[str]:
    """Extract property detail URLs from listing page.

    Args:
        html: HTML content of listing page

    Returns:
        List of property detail URLs
    """
    soup = BeautifulSoup(html, "lxml")
    urls = set()

    # Find all links to property pages
    # URL pattern: /annonces/achat/appartement/.../123456.htm
    for a in soup.find_all("a", href=True):
        href = a["href"]
        if "/annonces/" in href and ".htm" in href:
            # Clean URL (remove query parameters)
            url = href.split("?")[0].split("#")[0]
            if url.endswith(".htm"):
                # Make absolute URL
                full_url = urljoin(BASE_URL, url)
                urls.add(full_url)

    return list(urls)


def parse_property_details(html: str, url: str) -> dict:
    """Parse property details from detail page.

    Args:
        html: HTML content of property detail page
        url: Property URL

    Returns:
        Dictionary of property details
    """
    soup = BeautifulSoup(html, "lxml")
    details = {
        "url": url,
        "listing_id": extract_listing_id(url),
        "contract_type": extract_contract_type(url),
    }

    # Parse h1 heading for main info
    h1 = soup.find("h1")
    if h1:
        h1_text = h1.get_text(separator=" ", strip=True)

        # Property type
        details["property_type"] = parse_property_type(h1_text)

        # Price - look for price pattern
        price_match = re.search(r"([\d\s]+)\s*€", h1_text)
        if price_match:
            details["price"] = parse_price(price_match.group(0))

        # Price per sqm
        details["price_per_sqm"] = parse_price_per_sqm(h1_text)

        # Rooms, bedrooms, area
        details["rooms"] = parse_rooms(h1_text)
        details["bedrooms"] = parse_bedrooms(h1_text)
        details["living_area"] = parse_area(h1_text)

        # Floor
        floor_match = re.search(r"(RDC|[Éé]tage\s*\d+|\d+)/(\d+)", h1_text)
        if floor_match:
            floor, total = parse_floor(floor_match.group(0))
            details["floor"] = floor
            details["total_floors"] = total

    # Parse location from address section - look for patterns like "District, City (XXXXX)"
    # Try to find a button or div with address info containing postal code
    address_buttons = soup.find_all("button", string=re.compile(r"\(\d{5}\)"))
    address_divs = soup.find_all("div", string=re.compile(r"\(\d{5}\)"))
    address_generics = soup.find_all(string=re.compile(r"^[A-Za-zÀ-ÿ\s\-]+,\s*[A-Za-zÀ-ÿ\s\d]+\s*\(\d{5}\)$"))

    for elem in address_buttons + address_divs + list(address_generics):
        if hasattr(elem, 'get_text'):
            text = elem.get_text(strip=True)
        else:
            text = str(elem).strip()
        if text and re.search(r"\(\d{5}\)", text):
            # Only process if it looks like an address (not too long)
            if len(text) < 100:
                details["postal_code"] = parse_postal_code(text)
                details["city"] = parse_city(text)
                details["district"] = parse_district(text)
                break

    # Parse h2 title (property description title)
    h2_list = soup.find_all("h2")
    for h2 in h2_list:
        text = h2.get_text(strip=True)
        if text and "Caractéristiques" not in text and "Performance" not in text:
            if not text.startswith("Découvrez") and not text.startswith("Donnez"):
                details["title"] = text
                break

    # Parse description - look for the description text after the h2 title
    for h2 in h2_list:
        h2_text = h2.get_text(strip=True)
        # Skip standard sections
        if any(keyword in h2_text.lower() for keyword in ["caractéristiques", "performance", "découvrez", "donnez", "financement", "détails", "quartier", "copropriété", "géorisques", "partenaires", "perspectives", "visite"]):
            continue
        # This is likely the property title, get the next sibling for description
        next_elem = h2.find_next_sibling()
        if next_elem:
            desc_text = next_elem.get_text(strip=True)
            if len(desc_text) > 50 and len(desc_text) < 3000:
                # Filter out navigation/menu text
                if not any(keyword in desc_text.lower() for keyword in ["cookie", "confidentialité", "conditions", "seloger", "acheter", "louer", "vendre"]):
                    details["description"] = desc_text
                    break

    # Parse characteristics list
    char_section = soup.find("h2", string=re.compile(r"Caractéristiques"))
    if char_section:
        char_list = char_section.find_next("ul")
        if char_list:
            for li in char_list.find_all("li"):
                text = li.get_text(strip=True)
                text_lower = text.lower()

                if "cuisine" in text_lower:
                    details["kitchen_type"] = text
                elif "cave" in text_lower or "rangement" in text_lower:
                    details["storage"] = text
                elif "wc" in text_lower:
                    match = re.search(r"(\d+)\s*wc", text_lower)
                    if match:
                        details["wc"] = int(match.group(1))
                elif "balcon" in text_lower:
                    details["balcony"] = "pas de" not in text_lower
                elif "terrasse" in text_lower:
                    details["terrace"] = "pas de" not in text_lower
                elif "jardin" in text_lower:
                    details["garden"] = "pas de" not in text_lower
                elif "parking" in text_lower or "garage" in text_lower:
                    details["parking"] = "pas de" not in text_lower
                elif "ascenseur" in text_lower:
                    details["elevator"] = "pas de" not in text_lower and "sans" not in text_lower
                elif "exposition" in text_lower:
                    match = re.search(r"exposition[:\s]*(.+)", text, re.IGNORECASE)
                    if match:
                        details["exposure"] = match.group(1).strip()
                elif "calme" in text_lower or "lumineux" in text_lower:
                    details["ambiance"] = text
                elif "étage" in text_lower and "floor" not in str(details.get("floor", "")):
                    floor, total = parse_floor(text)
                    if floor:
                        details["floor"] = floor
                    if total:
                        details["total_floors"] = total
                elif "chambre" in text_lower and not details.get("bedrooms"):
                    details["bedrooms"] = parse_bedrooms(text)

    # Parse energy performance
    energy_section = soup.find("h2", string=re.compile(r"Performance énergétique"))
    if energy_section:
        parent = energy_section.find_parent()
        if parent:
            # DPE rating
            dpe_heading = parent.find("h3", string=re.compile(r"DPE|performance énergétique", re.IGNORECASE))
            if dpe_heading:
                # Look for rating letter (A-G)
                rating_elem = dpe_heading.find_next(string=re.compile(r"^[A-G]$"))
                if rating_elem:
                    details["energy_rating"] = rating_elem.strip()

            # GES rating
            ges_heading = parent.find("h3", string=re.compile(r"GES|gaz à effet de serre", re.IGNORECASE))
            if ges_heading:
                rating_elem = ges_heading.find_next(string=re.compile(r"^[A-G]$"))
                if rating_elem:
                    details["emissions_rating"] = rating_elem.strip()

            # Year built, heating, energy source from list
            info_list = parent.find("ul")
            if info_list:
                for li in info_list.find_all("li"):
                    text = li.get_text(strip=True)
                    if "Année de construction" in text:
                        match = re.search(r"(\d{4})", text)
                        if match:
                            details["year_built"] = match.group(1)
                    elif "chauffage" in text.lower():
                        # Extract value after colon or from last part
                        parts = text.split(":")
                        if len(parts) > 1:
                            details["heating_type"] = parts[-1].strip()
                        else:
                            details["heating_type"] = text.replace("Type de chauffage", "").strip()
                    elif "énergie" in text.lower() or "Sources" in text:
                        parts = text.split(":")
                        if len(parts) > 1:
                            details["energy_source"] = parts[-1].strip()
                        else:
                            details["energy_source"] = text.replace("Sources d'énergie", "").strip()

    # Parse price details section
    price_section = soup.find("h2", string=re.compile(r"Détails du prix"))
    if price_section:
        parent = price_section.find_parent()
        if parent:
            text = parent.get_text()

            # Price without fees
            match = re.search(r"Prix hors honoraires[:\s]*([\d\s]+)\s*€", text)
            if match:
                details["price_without_fees"] = parse_price(match.group(1))

            # Agency fees
            match = re.search(r"Honoraires[^€]*([\d\s]+)\s*€", text)
            if match:
                details["agency_fees"] = parse_price(match.group(1))

            # Agency fees percent
            match = re.search(r"\(([\d,\.]+)%[^)]*\)", text)
            if match:
                details["agency_fees_percent"] = float(match.group(1).replace(",", "."))

            # Notary fees
            match = re.search(r"Frais de notaire[^€]*([\d\s]+)\s*€", text)
            if match:
                details["notary_fees"] = parse_price(match.group(1))

            # Energy bill estimate
            match = re.search(r"entre\s*([\d\s]+)\s*et\s*([\d\s]+)\s*€/an", text)
            if match:
                details["energy_bill_min"] = parse_price(match.group(1))
                details["energy_bill_max"] = parse_price(match.group(2))

    # Parse copropriété info
    copro_section = soup.find("h2", string=re.compile(r"copropriété", re.IGNORECASE))
    if copro_section:
        parent = copro_section.find_parent()
        if parent:
            text = parent.get_text()

            # Number of lots
            match = re.search(r"Nombre de lots[:\s]*(\d+)", text)
            if match:
                details["copro_lots"] = int(match.group(1))

            # Charges
            match = re.search(r"Charges[^€]*([\d\s]+)\s*€/an", text)
            if match:
                details["copro_charges"] = parse_price(match.group(1))

            # Procedures
            if "pas de procédure" in text.lower():
                details["copro_procedures"] = "None"
            elif "procédure en cours" in text.lower():
                details["copro_procedures"] = "In progress"

    # Parse agency info
    agency_section = soup.find("h2", string=re.compile(r"Découvrez l'agence"))
    if agency_section:
        parent = agency_section.find_parent()
        if parent:
            agency_link = parent.find("a", href=re.compile(r"/professionnels/"))
            if agency_link:
                details["agency_name"] = agency_link.get_text(strip=True)

    # Alternative agency name from image alt
    if not details.get("agency_name"):
        agency_img = soup.find("img", alt=re.compile(r"Immobilier|Agence|Agency", re.IGNORECASE))
        if agency_img and agency_img.get("alt"):
            details["agency_name"] = agency_img["alt"]

    # Parse reference and ID from footer section
    id_section = soup.find(string=re.compile(r"Identifiant:"))
    if id_section:
        parent = id_section.find_parent()
        if parent:
            text = parent.get_text()
            # Reference
            match = re.search(r"Référence annonce[:\s]*(\S+)", text)
            if match:
                details["reference"] = match.group(1)

    return details