ytextractor.py

#!/usr/bin/env python3
"""
ytextractor.py
"""

import os
import re
import json
import hashlib
import logging
import sqlite3
from datetime import datetime, date
from typing import Optional, List, Dict, Any
import requests
import typer
from dotenv import load_dotenv
import yt_dlp # Import yt_dlp library
import sys # Added for sys.stdout.flush()
from functools import partial # Added for partial function application
import urllib.parse # Added for URL parsing

__version__ = "0.1.0"

app = typer.Typer()

DB_FILE = "yt.db"
CHANNELS_LIST_FILE = "channels.list"
THUMBNAILS_DIR = "thumbnails"
VIDEOS_DIR = "videos"

# Load environment variables from .env file
load_dotenv()

# --- Logging Configuration ---
LOG_LEVEL_MAP = {
    "DEBUG": logging.DEBUG,
    "INFO": logging.INFO,
    "WARNING": logging.WARNING,
    "ERROR": logging.ERROR,
    "CRITICAL": logging.CRITICAL,
}

# Get log level from environment variable, default to INFO
log_level_str = os.getenv("YTEXTRACTOR_LOG_LEVEL", "INFO").upper()
log_level = LOG_LEVEL_MAP.get(log_level_str, logging.INFO)

logging.basicConfig(
    filename="ytextractor.log",
    level=log_level,
    format="%(asctime)s %(levelname)s %(message)s",
)

# Configure yt-dlp specific loggers to suppress verbose output in the file
logging.getLogger('yt_dlp_logger').setLevel(logging.ERROR)
logging.getLogger('yt_dlp_adhoc_logger').setLevel(logging.ERROR)
logging.getLogger('yt_dlp_metadata_logger').setLevel(logging.ERROR)
# --- End Logging Configuration ---


API_KEY = os.getenv("YOUTUBE_API_KEY")

API_KEY_HASH: Optional[str] = None
if API_KEY:
    API_KEY_HASH = hashlib.sha256(API_KEY.encode()).hexdigest()
else:
    logging.error("YOUTUBE_API_KEY not found in environment. Please set it in your .env file.")

YT_API_BASE = "https://www.googleapis.com/youtube/v3"

# --- API Helper Functions ---

def _make_api_request(endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
    """Helper to make YouTube API requests and handle common errors."""
    if not API_KEY:
        typer.echo("Error: YOUTUBE_API_KEY not found in environment. Please set it in your .env file.")
        raise typer.Exit(1)
    
    params["key"] = API_KEY
    url = f"{YT_API_BASE}/{endpoint}"
    try:
        logging.debug(f"Making API request to: {url} with params: {params}")
        response = requests.get(url, params=params, timeout=10)
        response.raise_for_status()  # Raises HTTPError for bad responses (4xx or 5xx)
        logging.debug(f"API request to {endpoint} successful. Status: {response.status_code}")
        return response.json()
    except requests.exceptions.HTTPError as e:
        logging.error(f"HTTP error during API request to {endpoint}: {e} - {response.text}")
        typer.echo(f"Error: API request to {endpoint} failed with status {response.status_code}. Details: {response.text}")
        raise
    except requests.exceptions.ConnectionError as e:
        logging.error(f"Connection error during API request to {endpoint}: {e}")
        typer.echo(f"Error: Failed to connect to YouTube API at {endpoint}. Check your internet connection.")
        raise
    except requests.exceptions.Timeout as e:
        logging.error(f"Timeout error during API request to {endpoint}: {e}")
        typer.echo(f"Error: YouTube API request to {endpoint} timed out.")
        raise
    except requests.exceptions.RequestException as e:
        logging.error(f"An unexpected request error occurred to {endpoint}: {e}")
        typer.echo(f"Error: An unexpected error occurred during API request to {endpoint}.")
        raise
    except json.JSONDecodeError as e:
        logging.error(f"Failed to decode JSON from API response for {endpoint}: {e}")
        typer.echo(f"Error: Invalid JSON response from YouTube API at {endpoint}.")
        raise

def get_channel_info(channel_identifier: str) -> Optional[dict]:
    """
    Accepts a channel identifier (channel ID, @handle, or channel name).
    Returns channel info dict or None.
    """
    channel_identifier = channel_identifier.strip()
    logging.debug(f"Attempting to get channel info for: {channel_identifier}")

    if channel_identifier.startswith("UC"):  # Likely a channel ID
        params = {"part": "snippet,contentDetails", "id": channel_identifier}
        try:
            data = _make_api_request("channels", params)
            return data["items"][0] if data.get("items") else None
        except Exception as e:
            logging.warning(f"Could not fetch channel by ID {channel_identifier}: {e}")
            return None
    elif channel_identifier.startswith("@"):  # Handle
        handle = channel_identifier[1:]
        params = {"part": "snippet,contentDetails", "forHandle": handle}
        try:
            data = _make_api_request("channels", params)
            return data["items"][0] if data.get("items") else None
        except Exception as e:
            logging.warning(f"Could not fetch channel by handle @{handle}: {e}")
            # Fallback to search if direct handle lookup fails or is not supported everywhere (e.g., older API versions)
            search_params = {
                "part": "snippet",
                "q": handle,
                "type": "channel",
                "maxResults": 1,
            }
            try:
                search_data = _make_api_request("search", search_params)
                if search_data.get("items"):
                    channel_id = search_data["items"][0]["snippet"]["channelId"]
                    logging.info(f"Found channel ID {channel_id} via search for handle @{handle}.")
                    # Get full channel info using the ID found from search
                    return get_channel_info(channel_id)
            except Exception as se:
                logging.warning(f"Fallback search for handle @{handle} failed: {se}")
            return None
    else:  # Assume channel name or legacy username
        # Try as legacy username first
        params = {"part": "snippet,contentDetails", "forUsername": channel_identifier}
        try:
            data = _make_api_request("channels", params)
            if data.get("items"):
                logging.info(f"Found channel by username {channel_identifier}.")
                return data["items"][0]
        except Exception as e:
            logging.warning(f"Could not fetch channel by username {channel_identifier}: {e}")

        # Fallback to general search for channel name
        search_params = {
            "part": "snippet",
            "q": channel_identifier,
            "type": "channel",
            "maxResults": 1,
        }
        try:
            search_data = _make_api_request("search", search_params)
            if search_data.get("items"):
                # Prioritize channel results if any
                for item in search_data["items"]:
                    if item["id"]["kind"] == "youtube#channel":
                        channel_id = item["id"]["channelId"]
                        logging.info(f"Found channel ID {channel_id} via broad search for '{channel_identifier}'.")
                        return get_channel_info(channel_id)
                # If no channel results, but other types exist, just log
                if search_data.get("items"):
                    logging.info(f"Search for '{channel_identifier}' found non-channel results. Skipping.")
        except Exception as se:
            logging.warning(f"Fallback search for channel name '{channel_identifier}' failed: {se}")
        return None


def get_uploads_playlist_id(channel_id: str) -> Optional[str]:
    """Retrieves the uploads playlist ID for a given channel ID."""
    logging.debug(f"Getting uploads playlist ID for channel: {channel_id}")
    params = {"part": "contentDetails", "id": channel_id}
    try:
        data = _make_api_request("channels", params)
        items = data.get("items", [])
        if items:
            uploads_id = items[0]["contentDetails"]["relatedPlaylists"]["uploads"]
            logging.debug(f"Uploads playlist ID for {channel_id}: {uploads_id}")
            return uploads_id
    except (KeyError, IndexError) as e:
        logging.error(f"Could not parse uploads playlist ID for channel {channel_id}: {e}")
    except Exception as e:
        logging.error(f"Failed to get uploads playlist ID for channel {channel_id}: {e}")
    return None

def get_videos_from_playlist(playlist_id: str, page_token: Optional[str] = None):
    """Fetches a page of video items from a playlist."""
    logging.debug(f"Fetching videos from playlist: {playlist_id}, page_token: {page_token}")
    params = {
        "part": "snippet,contentDetails",
        "playlistId": playlist_id,
        "maxResults": 50,
    }
    if page_token:
        params["pageToken"] = page_token
    return _make_api_request("playlistItems", params)

def get_video_details_batch(video_ids: List[str]) -> Dict[str, Any]:
    """Fetches detailed video information for a list of video IDs."""
    if not video_ids:
        return {"items": []}
    logging.debug(f"Fetching details for video IDs batch: {video_ids}")
    params = {
        "part": "snippet,contentDetails",
        "id": ",".join(video_ids),
        "maxResults": 50, # Max results for video details is also 50
    }
    return _make_api_request("videos", params)

def download_thumbnail(url: str, filepath: str, force_download: bool = False):
    """Downloads a thumbnail image."""
    logging.debug(f"Attempting to download thumbnail from {url} to {filepath}")
    try:
        if os.path.exists(filepath) and not force_download:
            logging.info(f"Thumbnail already exists, skipping download: {filepath}")
            return
        
        # Ensure the directory exists before attempting to write the file
        os.makedirs(os.path.dirname(filepath), exist_ok=True)

        r = requests.get(url, stream=True, timeout=5)
        r.raise_for_status()
        with open(filepath, "wb") as f:
            for chunk in r.iter_content(chunk_size=8192):
                f.write(chunk)
        logging.info(f"Downloaded thumbnail to {filepath} (forced: {force_download})")
    except requests.exceptions.RequestException as e:
        logging.error(f"Failed to download thumbnail {url}: {e}")
    except IOError as e:
        logging.error(f"File I/O error while saving thumbnail {filepath}: {e}")
    except Exception as e:
        logging.error(f"An unexpected error occurred downloading thumbnail {url}: {e}")

def _progress_hook_on_line(d, current_video_num: Optional[int] = None, total_videos_num: Optional[int] = None):
    """
    Custom progress hook for yt-dlp to print updates on the same line.
    Includes overall video count if provided.
    """
    prefix = ""
    if current_video_num is not None and total_videos_num is not None:
        prefix = f"[{current_video_num} / {total_videos_num}]: "

    if d['status'] == 'downloading':
        total_bytes_str = d.get('total_bytes_str', 'N/A')
        downloaded_bytes_str = d.get('downloaded_bytes_str', 'N/A')
        percent_str = d.get('_percent_str', 'N/A')
        speed_str = d.get('_speed_str', 'N/A')
        eta_str = d.get('_eta_str', 'N/A')

        output_parts = [f"\rDownloading {prefix}{percent_str}"]
        if total_bytes_str != 'N/A': # Only add "of [total size]" if total size is known
            output_parts.append(f" of {total_bytes_str}")
        output_parts.append(f" at {speed_str} ETA {eta_str}    ")

        typer.echo("".join(output_parts), nl=False)
        sys.stdout.flush() # Ensure it's written immediately
    elif d['status'] == 'finished':
        typer.echo("") # Print a newline to move to the next line after download finishes

def download_video(video_id: str, title: str, published_at: str, duration: str, channel_handle: str, preset_value: Optional[str] = None, current_video_num: Optional[int] = None, total_videos_num: Optional[int] = None) -> bool:
    """
    Downloads a video using yt-dlp to the structured videos directory.
    This function is used by the 'fetch-videos' command and interacts with pre-fetched metadata.
    Returns True if downloaded or already exists, False otherwise.
    """
    date_prefix = iso8601_to_date(published_at) # Uses API's ISO date
    safe_title = sanitize_filename(title)
    parsed_duration = parse_iso8601_duration(duration) # Uses API's ISO duration
    
    filename_suffix = f"_{preset_value}" if preset_value and preset_value != "best" else "" # Don't add "_best" suffix
    # Construct filename: YYMMDD_video name_video duration_videoid_preset.mp4
    filename = f"{date_prefix}_{safe_title}_{parsed_duration}_{video_id}{filename_suffix}.mp4"
    
    channel_video_folder = os.path.join(VIDEOS_DIR, sanitize_filename(channel_handle))
    os.makedirs(channel_video_folder, exist_ok=True)
    
    filepath = os.path.join(channel_video_folder, filename)

    if os.path.exists(filepath):
        logging.info(f"Video already exists, skipping download: {filepath}")
        return True

    # Define yt-dlp format based on preset
    format_string = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]' # Default: best quality MP4
    if preset_value == 'normal':
        format_string = 'bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720][ext=mp4]' # Up to 720p
    elif preset_value == 'small':
        format_string = 'bestvideo[height<=360][ext=mp4]+bestaudio[ext=m4a]/best[height<=360][ext=mp4]' # Up to 360p

    # Create a partial function for the progress hook to pass additional arguments
    progress_hook_partial = partial(_progress_hook_on_line, current_video_num=current_video_num, total_videos_num=total_videos_num)

    ydl_opts = {
        'format': format_string,
        'outtmpl': filepath,
        'merge_output_format': 'mp4',
        'noplaylist': True,
        'ignoreerrors': True, # Continue on download errors for individual videos
        'quiet': True,
        'no_warnings': True,
        'progress_hooks': [progress_hook_partial], # Use our custom hook with context
        'postprocessors': [{
            'key': 'FFmpegVideoConvertor',
            'preferedformat': 'mp4',
        }],
        'logger': logging.getLogger('yt_dlp_logger') # Route yt-dlp internal logs to our logger
    }

    try:
        logging.info(f"Attempting to download video {video_id}: {title} to {filepath} with preset '{preset_value}'")
        # yt-dlp expects a real YouTube URL or ID. Using the direct video ID here is better.
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            ydl.download([video_id]) # Use video_id directly
        logging.info(f"Successfully downloaded video {video_id}: {title} with preset '{preset_value}'")
        return True
    except yt_dlp.utils.DownloadError as e:
        logging.error(f"yt-dlp download error for {video_id} ({title}) with preset '{preset_value}': {e}")
        typer.echo(f"\nError downloading video {video_id} ({title}) with preset '{preset_value}': {e}") # Add newline
    except Exception as e:
        logging.error(f"An unexpected error occurred during video download for {video_id} ({title}) with preset '{preset_value}': {e}")
        typer.echo(f"\nAn unexpected error occurred downloading video {video_id} ({title}) with preset '{preset_value}': {e}") # Add newline
    return False

def download_video_adhoc(video_identifier: str, target_dir: str, preset_value: str) -> bool:
    """
    Downloads a single video by URL or ID to a specified directory.
    This function is used by the standalone 'download' command and fetches its own metadata.
    Returns True on success, False on failure.
    """
    
    # --- MODIFICATION START ---
    # First, extract the pure video ID from the identifier
    video_id = _extract_video_id(video_identifier)
    if not video_id:
        typer.echo(f"Error: Could not extract a valid YouTube video ID from '{video_identifier}'. Please check the URL or ID.")
        logging.error(f"Failed to extract video ID from ad-hoc download identifier: {video_identifier}")
        return False
    # --- MODIFICATION END ---
    
    logging.info(f"Attempting ad-hoc download of video ID '{video_id}' (from '{video_identifier}') to {target_dir} with preset '{preset_value}'")
    

    # Ensure target directory exists
    os.makedirs(target_dir, exist_ok=True)

    # Now, get video metadata using yt-dlp using the extracted video_id
    video_info = _get_yt_dlp_video_metadata(video_id) # Use the extracted video_id
    if not video_info:
        typer.echo(f"Error: Could not retrieve video information for {video_id}. Check ID.")
        logging.error(f"Failed to get metadata for ad-hoc download: {video_id}")
        return False

    title = video_info.get("title", "Untitled_Video")
    published_date_yymmdd = _format_yymmdd_for_filename_from_yyyymmdd(video_info.get("published_at", ""))
    duration_str = _format_duration_for_filename_from_seconds(video_info.get("duration", 0))
    
    safe_title = sanitize_filename(title)
    
    filename_suffix = f"_{preset_value}" if preset_value != "best" else "" # Don't add "_best" suffix
    filename = f"{published_date_yymmdd}_{safe_title}_{duration_str}_{video_id}{filename_suffix}.mp4"
    filepath = os.path.join(target_dir, filename)

    if os.path.exists(filepath):
        typer.echo(f"Video already exists, skipping download: {filepath}")
        logging.info(f"Ad-hoc video already exists, skipping download: {filepath}")
        return True

    # Define yt-dlp format based on preset
    format_string = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]' # Default: best quality MP4
    if preset_value == 'normal':
        format_string = 'bestvideo[height<=720][ext=mp4]+bestaudio[ext=m4a]/best[height<=720][ext=mp4]' # Up to 720p
    elif preset_value == 'small':
        format_string = 'bestvideo[height<=360][ext=mp4]+bestaudio[ext=m4a]/best[height<=360][ext=mp4]' # Up to 360p

    ydl_opts = {
        'format': format_string,
        'outtmpl': filepath,
        'merge_output_format': 'mp4',
        'noplaylist': True,
        'ignoreerrors': False,
        'quiet': True, # Set to True to suppress yt-dlp's default progress output
        'no_warnings': True, # Set to True to suppress yt-dlp's warnings
        'progress_hooks': [_progress_hook_on_line], # Use our custom hook for in-line updates (without video count)
        'postprocessors': [{
            'key': 'FFmpegVideoConvertor',
            'preferedformat': 'mp4',
        }],
        'logger': logging.getLogger('yt_dlp_adhoc_logger') # Dedicated logger for ad-hoc
    }

    typer.echo(f"Attempting to download '{title}' ({video_id}) to '{filepath}' with preset '{preset_value}'...")
    logging.info(f"Starting ad-hoc yt-dlp download for {video_id} with format {format_string}.")

    try:
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            ydl.download([video_id]) # Use the extracted video_id here
        typer.echo(f"\nSuccessfully downloaded: {filepath}") # Newline after progress finishes
        logging.info(f"Ad-hoc download successful: {filepath}")
        return True
    except yt_dlp.utils.DownloadError as e:
        typer.echo(f"\nError: Failed to download video {video_id}. Details: {e}")
        logging.error(f"Ad-hoc yt-dlp download error for {video_id}: {e}")
    except Exception as e:
        typer.echo(f"\nAn unexpected error occurred during download: {e}")
        logging.error(f"An unexpected error occurred during ad-hoc download for {video_id}: {e}")
    return False


# --- Utility Functions ---

def parse_iso8601_duration(duration_str: str) -> str:
    """Converts ISO 8601 duration string (e.g., PT1H2M3S) to a compact filename format (e.g., 1h02m03s).
       Used for YouTube API-sourced durations."""
    if not duration_str or not duration_str.startswith('PT'):
        return ""

    duration_str = duration_str[2:] # Remove 'PT'

    # Regex to find numbers followed by H, M, or S
    parts = re.findall(r'(\d+)([HMS])', duration_str)

    hours = 0
    minutes = 0
    seconds = 0

    for value, unit in parts:
        value = int(value)
        if unit == 'H':
            hours = value
        elif unit == 'M':
            minutes = value
        elif unit == 'S':
            seconds = value

    # Format for filename: HHhMMmSSs, MMmSSs, or SSs
    formatted_duration_parts = []

    if hours > 0:
        formatted_duration_parts.append(f"{hours}h")
        # Ensure minutes and seconds are always two digits if hours are present
        formatted_duration_parts.append(f"{minutes:02d}m")
        formatted_duration_parts.append(f"{seconds:02d}s")
    elif minutes > 0:
        formatted_duration_parts.append(f"{minutes}m")
        # Ensure seconds are two digits if minutes are present
        formatted_duration_parts.append(f"{seconds:02d}s")
    elif seconds > 0:
        formatted_duration_parts.append(f"{seconds}s")
    else: # Case for "PT0S" or empty after PT
        return "00s" # Default to "00s" for zero duration

    return "".join(formatted_duration_parts)

def _format_duration_for_filename_from_seconds(total_seconds: Optional[int]) -> str:
    """Converts total seconds (from yt-dlp) to a compact filename format (e.g., 1h02m03s)."""
    if total_seconds is None:
        return "00s" # Default for unknown duration

    hours = total_seconds // 3600
    minutes = (total_seconds % 3600) // 60
    seconds = total_seconds % 60

    formatted_parts = []
    if hours > 0:
        formatted_parts.append(f"{hours}h")
        formatted_parts.append(f"{minutes:02d}m")
        formatted_parts.append(f"{seconds:02d}s")
    elif minutes > 0:
        formatted_parts.append(f"{minutes}m")
        formatted_parts.append(f"{seconds:02d}s")
    elif seconds > 0:
        formatted_parts.append(f"{seconds}s")
    else:
        return "00s"
    return "".join(formatted_parts)

def sanitize_filename(filename: str) -> str:
    """Removes or replaces characters not allowed in filenames."""
    # Replace characters that are typically problematic in filenames
    filename = re.sub(r'[\\/:*?"<>|]', '', filename)
    # Replace multiple spaces with a single underscore, trim leading/trailing spaces/underscores
    filename = re.sub(r'\s+', '_', filename).strip('_')
    # Limit filename length if necessary, though unlikely for video titles
    return filename

def iso8601_to_date(iso_str: str) -> str:
    """Converts ISO8601 string to YYMMDD format. Used for YouTube API-sourced dates."""
    try:
        # datetime.fromisoformat handles 'Z' by interpreting it as UTC, then converting to local if not specified
        # explicitly making it UTC aware first helps
        dt = datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
        return dt.strftime("%y%m%d")
    except ValueError:
        logging.warning(f"Could not parse date string: {iso_str}. Returning empty string.")
        return ""

def _format_yymmdd_for_filename_from_yyyymmdd(yyyymmdd_str: str) -> str:
    """ConvertsYYYYMMDD string (from yt-dlp's upload_date) to YYMMDD format."""
    if not yyyymmdd_str:
        return ""
    try:
        dt = datetime.strptime(yyyymmdd_str, "%Y%m%d")
        return dt.strftime("%y%m%d")
    except ValueError:
        logging.warning(f"Could not parseYYYYMMDD date string from yt-dlp: {yyyymmdd_str}. Returning empty string.")
        return ""

def _parse_yymmdd_to_datetime(date_str: str) -> Optional[date]:
    """Parses a YYMMDD string into a datetime.date object."""
    try:
        # Use strptime with %y for 2-digit year (00-99). Assumes 20xx for 00-99.
        # This will correctly handle YY (e.g., 24 for 2024, 99 for 1999 if within epoch)
        # For dates like 900101, it will default to 1990-01-01. For 240101, it will be 2024-01-01.
        return datetime.strptime(date_str, "%y%m%d").date()
    except ValueError:
        logging.error(f"Invalid cutoff date format: '{date_str}'. Expected YYMMDD (e.g., 231231 for Dec 31, 2023).")
        typer.echo(f"Error: Invalid cutoff date format '{date_str}'. Expected YYMMDD (e.g., 231231). Skipping cutoff.")
        return None

def _get_yt_dlp_video_metadata(url_or_id: str) -> Optional[Dict[str, Any]]:
    """
    Uses yt-dlp to extract video metadata without downloading.
    Returns a dictionary of relevant info or None on failure.
    """
    ydl_opts = {
        'quiet': True,
        'no_warnings': True,
        'skip_download': True,
        'force_generic_extractor': True, # Helps with direct IDs
        'logger': logging.getLogger('yt_dlp_metadata_logger')
    }
    try:
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            info = ydl.extract_info(url_or_id, download=False)
            # yt-dlp's 'upload_date' isYYYYMMDD
            # yt-dlp's 'duration' is in seconds (int)
            # 'channel_handle' might be present, or derived from 'uploader_url'
            return {
                'video_id': info.get('id'),
                'title': info.get('title'),
                'published_at': info.get('upload_date'),
                'duration': info.get('duration'), # integer seconds
                'channel_handle': info.get('channel_handle') or info.get('uploader_url', '').split('/')[-1] if info.get('uploader_url') else 'unknown_channel'
            }
    except yt_dlp.utils.DownloadError as e:
        logging.error(f"yt-dlp error fetching metadata for {url_or_id}: {e}")
        return None
    except Exception as e:
        logging.error(f"An unexpected error occurred getting metadata for {url_or_id}: {e}")
        return None

def _extract_video_id(url_or_id: str) -> Optional[str]:
    """
    Extracts the YouTube video ID from a given URL or string.
    Prioritizes 'v' parameter in query string, then falls back to yt-dlp's extraction.
    """
    # 1. Try to parse as URL and get 'v' parameter
    try:
        parsed_url = urllib.parse.urlparse(url_or_id)
        query_params = urllib.parse.parse_qs(parsed_url.query)
        if 'v' in query_params and query_params['v'][0]:
            logging.debug(f"Extracted video ID '{query_params['v'][0]}' from 'v' parameter of URL: {url_or_id}")
            return query_params['v'][0]
        
        # For youtu.be links, the video ID is directly the path without the leading slash
        if parsed_url.netloc == 'youtu.be' and parsed_url.path:
            video_id_from_path = parsed_url.path.lstrip('/')
            # Simple check for typical YouTube ID length/chars, could be more robust
            if re.match(r'^[a-zA-Z0-9_-]{11}$', video_id_from_path):
                logging.debug(f"Extracted video ID '{video_id_from_path}' from youtu.be path: {url_or_id}")
                return video_id_from_path

    except Exception as e:
        logging.warning(f"Failed to parse URL or extract 'v' parameter from '{url_or_id}' directly: {e}")
        # Fall through to yt-dlp if direct parsing fails

    # 2. Fall back to yt-dlp's info extraction if direct parsing didn't find a 'v' or wasn't a well-formed URL
    ydl_opts = {
        'quiet': True,
        'no_warnings': True,
        'extract_flat': True, # Only extract basic info, don't recurse into playlists
        'force_generic_extractor': True,
        'logger': logging.getLogger('yt_dlp_metadata_logger') # Use the metadata logger
    }
    try:
        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
            # yt-dlp's extract_info can parse various URL types and return the video ID
            info = ydl.extract_info(url_or_id, download=False, process=False) # process=False to avoid full processing
            if info and 'id' in info:
                logging.debug(f"Extracted video ID '{info['id']}' using yt-dlp for: {url_or_id}")
                return info.get('id')
    except yt_dlp.utils.DownloadError as e:
        logging.error(f"yt-dlp error extracting video ID from '{url_or_id}': {e}")
        return None
    except Exception as e:
        logging.error(f"An unexpected error occurred extracting video ID from '{url_or_id}': {e}")
        return None
    return None # Return None if no ID could be extracted by any method

# --- Database Functions ---

def init_db():
    """Initializes the SQLite database tables."""
    conn = sqlite3.connect(DB_FILE)
    c = conn.cursor()

    c.execute(
        """
        CREATE TABLE IF NOT EXISTS channels (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            channel_id TEXT UNIQUE,
            channel_name TEXT,
            channel_handle TEXT
        )
        """
    )

    c.execute(
        """
        CREATE TABLE IF NOT EXISTS videos (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            video_id TEXT UNIQUE,
            channel_id TEXT,
            title TEXT,
            description TEXT,
            published_at TEXT,
            duration TEXT,
            thumbnail_url TEXT,
            raw_data TEXT
        )
        """
    )

    c.execute(
        """
        CREATE TABLE IF NOT EXISTS fetch_log (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            run_timestamp TEXT,
            total_channels_processed INTEGER,
            total_new_videos INTEGER,
            total_videos_in_db INTEGER,
            total_api_calls INTEGER,
            api_key_hash TEXT,
            details TEXT
        )
        """
    )

    conn.commit()
    return conn

# --- Main Logic ---

def fetch_channel_videos(channel_identifier: str, conn: sqlite3.Connection, force_all: bool, cutoff_date_str: Optional[str] = None) -> dict:
    """
    Fetches videos from a channel's uploads playlist, batches API calls for video details,
    and stores data in the DB.
    Returns dict with fetch statistics.
    """
    c = conn.cursor()

    ch_info = get_channel_info(channel_identifier)
    if not ch_info:
        typer.echo(f"Could not find channel info for {channel_identifier}. Skipping.")
        logging.error(f"Failed to get channel info for {channel_identifier}.")
        return {}

    channel_id = ch_info["id"]
    snippet = ch_info.get("snippet", {})
    channel_name = snippet.get("title", "Unknown Channel")
    channel_handle = snippet.get("customUrl", "").lstrip('/')
    if not channel_handle:
        channel_handle = f"@{channel_name.replace(' ', '').lower()}"

    logging.info(f"Processing channel: {channel_name} ({channel_id}, @{channel_handle})")
    typer.echo(f"Processing channel: {channel_name} (@{channel_handle})")

    # Insert or update channel in DB
    c.execute(
        """
        INSERT OR IGNORE INTO channels (channel_id, channel_name, channel_handle)
        VALUES (?, ?, ?)
        """,
        (channel_id, channel_name, channel_handle),
    )
    conn.commit()

    uploads_playlist_id = get_uploads_playlist_id(channel_id)
    if not uploads_playlist_id:
        typer.echo(f"Could not get uploads playlist for channel {channel_id}. Skipping.")
        logging.error(f"No uploads playlist found for channel {channel_id}.")
        return {}

    # Parse cutoff date if provided
    cutoff_date: Optional[date] = None
    if cutoff_date_str:
        cutoff_date = _parse_yymmdd_to_datetime(cutoff_date_str)
        if cutoff_date:
            logging.info(f"Applying cutoff date: {cutoff_date.strftime('%Y-%m-%d')} for channel {channel_name}.")
        else:
            logging.warning(f"Invalid cutoff date string '{cutoff_date_str}' provided for channel {channel_name}. Proceeding without cutoff.")

    new_videos_count = 0
    total_api_calls = 0
    next_page_token = None
    
    # Collect all new/to-be-forced video IDs and their basic details first
    video_ids_to_process_details = [] # This will hold IDs that are either new OR need to be re-processed due to force_all

    logging.info(f"Scanning playlist {uploads_playlist_id} for new/updated video IDs for channel {channel_name}.")
    typer.echo(f"Scanning for new/updated videos for {channel_name}.")
    while True:
        try:
            pl_data = get_videos_from_playlist(uploads_playlist_id, next_page_token)
            total_api_calls += 1
            if not pl_data.get("items"):
                logging.debug("No more items in playlist or end of playlist reached.")
                break # No more items
            
            for item in pl_data["items"]:
                video_id = item["contentDetails"]["videoId"]
                video_published_at_str = item["snippet"]["publishedAt"]
                
                # Check against cutoff date
                if cutoff_date:
                    try:
                        video_published_date = datetime.fromisoformat(video_published_at_str.replace("Z", "+00:00")).date()
                        if video_published_date < cutoff_date:
                            logging.info(f"Skipping video {video_id} ('{item['snippet']['title']}') as it is older than cutoff date {cutoff_date.strftime('%Y-%m-%d')}.")
                            continue # Skip this video if older than cutoff
                    except ValueError:
                        logging.warning(f"Could not parse published date '{video_published_at_str}' for video {video_id}. Not applying cutoff.")


                # Check if video exists in DB
                c.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
                exists_in_db = c.fetchone()
                
                # If it doesn't exist OR force_all is True, add it to our list for detail fetching
                if not exists_in_db or force_all:
                    video_ids_to_process_details.append(video_id)
            
            next_page_token = pl_data.get("nextPageToken")
            if not next_page_token:
                logging.debug("No next page token found. End of playlist.")
                break # No more pages
            else:
                logging.debug(f"Next page token: {next_page_token}")
        except Exception as e:
            logging.error(f"Error fetching playlist items for {channel_name}: {e}")
            typer.echo(f"Error fetching playlist items for {channel_name}: {e}")
            break
    
    logging.info(f"Found {len(video_ids_to_process_details)} new or updated video IDs to process details for {channel_name}.")
    if not video_ids_to_process_details:
        typer.echo(f"No new or updated videos found for {channel_name}. Skipping detail fetch and download.")
        c.execute("SELECT COUNT(*) FROM videos WHERE channel_id = ?", (channel_id,))
        total_videos_in_db_for_channel = c.fetchone()[0]
        return {
            "channel_id": channel_id,
            "channel_handle": channel_handle,
            "channel_name": channel_name,
            "new_videos_count": 0,
            "total_videos_in_db": total_videos_in_db_for_channel,
            "api_calls": total_api_calls,
        }

    # Fetch detailed video info for all identified videos
    all_new_video_details = []
    FETCH_DETAIL_BATCH_SIZE = 50 # Max results for video details is also 50
    for i in range(0, len(video_ids_to_process_details), FETCH_DETAIL_BATCH_SIZE):
        batch_ids = video_ids_to_process_details[i : i + FETCH_DETAIL_BATCH_SIZE]
        try:
            batch_details = get_video_details_batch(batch_ids)
            total_api_calls += 1
            all_new_video_details.extend(batch_details.get("items", []))
            logging.debug(f"Fetched details for batch of {len(batch_ids)} videos.")
        except Exception as e:
            logging.error(f"Error fetching video details for batch {batch_ids}: {e}")
            typer.echo(f"Error fetching video details for a batch of videos: {e}")
            continue

    # Sort the collected video details by publishedAt (oldest first)
    # This ensures older videos are inserted first, which helps with resumption on crash.
    all_new_video_details_sorted = sorted(
        all_new_video_details,
        key=lambda x: x.get('snippet', {}).get('publishedAt', ''), # Use empty string as default for sorting
        reverse=False # Oldest first
    )
    
    typer.echo(f"Processing details and downloading thumbnails for {len(all_new_video_details_sorted)} videos (oldest first).")

    # Now, process and insert/download in batches, oldest-first
    INSERT_PROCESS_BATCH_SIZE = 50 # Commit after processing this many videos
    for i in range(0, len(all_new_video_details_sorted), INSERT_PROCESS_BATCH_SIZE):
        current_insert_process_batch = all_new_video_details_sorted[i : i + INSERT_PROCESS_BATCH_SIZE]
        
        for video_info in current_insert_process_batch:
            video_id = video_info["id"]

            v_snippet = video_info.get("snippet", {})
            v_content = video_info.get("contentDetails", {})

            published_at = v_snippet.get("publishedAt", "")
            title = v_snippet.get("title", "No Title")
            description = v_snippet.get("description", "")
            duration = v_content.get("duration", "")
            thumbnails = v_snippet.get("thumbnails", {})

            thumbnail_url = (
                thumbnails.get("maxres", {}).get("url")
                or thumbnails.get("standard", {}).get("url")
                or thumbnails.get("high", {}).get("url")
                or thumbnails.get("medium", {}).get("url")
                or thumbnails.get("default", {}).get("url")
                or ""
            )
            raw_data = json.dumps(video_info)

            # Check if video existed previously for counting new_videos_count
            c.execute("SELECT 1 FROM videos WHERE video_id = ?", (video_id,))
            exists_in_db = c.fetchone()
            
            c.execute(
                """
                INSERT OR REPLACE INTO videos
                (video_id, channel_id, title, description, published_at, duration, thumbnail_url, raw_data)
                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                """,
                (
                    video_id,
                    channel_id,
                    title,
                    description,
                    published_at,
                    duration,
                    thumbnail_url,
                    raw_data,
                ),
            )

            # Increment new_videos_count only if it was genuinely a new addition (not an an update)
            if not exists_in_db:
                new_videos_count += 1
                logging.debug(f"New video added: {title} ({video_id})")
            else:
                logging.debug(f"Video updated/re-processed: {title} ({video_id})")


            # Download thumbnail
            if thumbnail_url:
                date_prefix = iso8601_to_date(published_at)
                safe_title = sanitize_filename(title)
                
                # Use the new parsed duration for the filename
                parsed_duration = parse_iso8601_duration(duration) 
                
                filename = f"{date_prefix}_{safe_title}_{parsed_duration}_{video_id}.jpg"
                channel_folder = os.path.join(THUMBNAILS_DIR, sanitize_filename(channel_handle))
                os.makedirs(channel_folder, exist_ok=True) # Ensure directory exists
                filepath = os.path.join(channel_folder, filename)
                download_thumbnail(thumbnail_url, filepath, force_all) # Pass force_all to force re-download if needed
        
        # Commit after each batch of insertions and thumbnail downloads to save progress
        conn.commit()
        logging.info(f"Committed {len(current_insert_process_batch)} videos for {channel_name}.")


    c.execute("SELECT COUNT(*) FROM videos WHERE channel_id = ?", (channel_id,))
    total_videos_in_db_for_channel = c.fetchone()[0]

    logging.info(f"Finished processing {channel_name}. New videos: {new_videos_count}, Total in DB: {total_videos_in_db_for_channel}.")

    return {
        "channel_id": channel_id,
        "channel_handle": channel_handle,
        "channel_name": channel_name,
        "new_videos_count": new_videos_count,
        "total_videos_in_db": total_videos_in_db_for_channel,
        "api_calls": total_api_calls,
    }

# --- Typer Commands ---

@app.command()
def fetch(
    channel: Optional[str] = typer.Argument(
        None,
        help="YouTube channel identifier (ID, @handle, or username). If omitted, reads from channels.list file.",
    ),
    force_all: bool = typer.Option(
        False,
        "--force-all",
        "-f",
        help="Force fetch all videos from the beginning and overwrite existing ones in the database and re-download thumbnails.",
    ),
    cutoff: Optional[str] = typer.Option(
        None,
        "--cutoff",
        "-c",
        help="Exclude videos published before this date (YYMMDD format).",
    ),
):
    """
    Fetch videos metadata and thumbnails for one or multiple YouTube channels.
    """
    conn = init_db()
    total_channels_processed = 0
    total_new_videos_across_all_channels = 0
    total_videos_in_db_overall = 0
    total_api_calls_made = 0
    fetch_details_log: List[Dict[str, Any]] = []

    try:
        channels_to_process = []
        if channel:
            channels_to_process.append(channel)
        else:
            if not os.path.isfile(CHANNELS_LIST_FILE):
                typer.echo(f"Channel list file '{CHANNELS_LIST_FILE}' not found. Please provide a channel or create the file.")
                raise typer.Exit(1)
            with open(CHANNELS_LIST_FILE, "r", encoding="utf-8") as f:
                channels_to_process = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]

        typer.echo(f"Starting fetch for {len(channels_to_process)} channel(s)...")
        logging.info(f"Starting fetch operation for {len(channels_to_process)} channel(s).")

        for ch_identifier in channels_to_process:
            try:
                result = fetch_channel_videos(ch_identifier, conn, force_all, cutoff) # Pass cutoff here
                if result:
                    total_channels_processed += 1
                    total_new_videos_across_all_channels += result.get("new_videos_count", 0)
                    total_api_calls_made += result.get("api_calls", 0)
                    fetch_details_log.append(
                        {
                            "channel_id": result.get("channel_id"),
                            "channel_handle": result.get("channel_handle"),
                            "channel_name": result.get("channel_name"),
                            "new_videos_fetched": result.get("new_videos_count"),
                            "total_videos_in_db_for_channel": result.get("total_videos_in_db"),
                            "api_calls_for_channel": result.get("api_calls"),
                        }
                    )
                else:
                    logging.warning(f"Failed to process channel {ch_identifier}. No results returned.")
            except Exception as e:
                logging.critical(f"Critical error processing channel {ch_identifier}: {e}", exc_info=True)
                typer.echo(f"Critical error processing channel {ch_identifier}. See log for details.")

        c = conn.cursor()
        c.execute("SELECT COUNT(*) FROM videos")
        total_videos_in_db_overall = c.fetchone()[0]
        
        c.execute(
            """
            INSERT INTO fetch_log
            (run_timestamp, total_channels_processed, total_new_videos, total_videos_in_db, total_api_calls, api_key_hash, details)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            """,
            (
                datetime.utcnow().isoformat(),
                total_channels_processed,
                total_new_videos_across_all_channels,
                total_videos_in_db_overall,
                total_api_calls_made,
                API_KEY_HASH,
                json.dumps(fetch_details_log),
            ),
        )
        conn.commit()

        typer.echo("\n--- Fetch Summary ---")
        typer.echo(f"Channels processed: {total_channels_processed}")
        typer.echo(f"New videos added: {total_new_videos_across_all_channels}")
        typer.echo(f"Total videos in DB: {total_videos_in_db_overall}")
        typer.echo(f"Total API calls made: {total_api_calls_made}")
        typer.echo("API Quota Status: Please check your Google Cloud Console for accurate status.")
        typer.echo(f"Detailed log available in {os.path.basename(logging.getLogger().handlers[0].baseFilename)}")

    except KeyboardInterrupt:
        typer.echo("\nFetch operation interrupted by user (CTRL+C). Exiting gracefully.")
        logging.info("Fetch operation interrupted by user (CTRL+C).")
        raise typer.Exit(0)
    except Exception as e:
        logging.critical(f"An unhandled error occurred during fetch: {e}", exc_info=True)
        typer.echo(f"An unexpected error occurred: {e}. Check log for details.")
        raise typer.Exit(1)
    finally:
        if conn:
            conn.close()
            logging.info("Database connection closed.")

@app.command(name="fetch-videos")
def fetch_videos_command(
    preset: Optional[str] = typer.Option(
        None,
        "--preset",
        "-p",
        help="Download video quality preset (best, normal, small). 'best' is default if not specified.",
    ),
):
    """
    Downloads videos from channels listed in channels.list if they are not already present.
    Videos are named YYMMDD_video name_video duration_videoid_preset.mp4 and saved in videos/@channel_handle/
    """
    conn = init_db()
    c = conn.cursor()
    total_videos_checked_overall = 0 
    total_videos_downloaded_overall = 0 
    channels_processed_count = 0

    valid_presets = ["best", "normal", "small"]
    if preset and preset not in valid_presets:
        typer.echo(f"Error: Invalid preset '{preset}'. Valid options are: {', '.join(valid_presets)}")
        raise typer.Exit(1)
    
    # Set default preset to 'best' if not specified
    actual_preset = preset if preset else "best"

    try:
        if not os.path.isfile(CHANNELS_LIST_FILE):
            typer.echo(f"Channel list file '{CHANNELS_LIST_FILE}' not found. Cannot fetch videos.")
            raise typer.Exit(1)
        
        with open(CHANNELS_LIST_FILE, "r", encoding="utf-8") as f:
            channel_identifiers = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]

        typer.echo(f"Starting video download process for {len(channel_identifiers)} channel(s) with preset '{actual_preset}'...")
        logging.info(f"Starting video download process for {len(channel_identifiers)} channel(s) with preset '{actual_preset}'.")

        for ch_identifier in channel_identifiers:
            channels_processed_count += 1
            channel_info_db = c.execute(
                "SELECT channel_id, channel_name, channel_handle FROM channels WHERE channel_id = (SELECT channel_id FROM channels WHERE channel_handle = ? OR channel_id = ? LIMIT 1) OR channel_name = ? LIMIT 1",
                (ch_identifier.lstrip('@'), ch_identifier, ch_identifier)
            ).fetchone()

            channel_id = None
            channel_name = None
            channel_handle = None

            if channel_info_db:
                channel_id, channel_name, channel_handle = channel_info_db
                typer.echo(f"\nProcessing videos for channel: {channel_name} (@{channel_handle})")
                logging.info(f"Processing videos for channel: {channel_name} (@{channel_handle}) [ID: {channel_id}]")
            else:
                # If not found in DB, try to get info via API (this will consume quota)
                ch_api_info = get_channel_info(ch_identifier)
                if ch_api_info:
                    channel_id = ch_api_info["id"]
                    channel_name = ch_api_info["snippet"]["title"]
                    channel_handle = ch_api_info["snippet"].get("customUrl", "").lstrip('/')
                    if not channel_handle:
                        channel_handle = f"@{channel_name.replace(' ', '').lower()}"
                    
                    # Update DB with channel info if found via API
                    c.execute(
                        """
                        INSERT OR IGNORE INTO channels (channel_id, channel_name, channel_handle)
                        VALUES (?, ?, ?)
                        """,
                        (channel_id, channel_name, channel_handle),
                    )
                    conn.commit()
                    typer.echo(f"\nDiscovered and processing videos for new channel: {channel_name} (@{channel_handle})")
                    logging.info(f"Discovered and processing videos for new channel: {channel_name} (@{channel_handle}) [ID: {channel_id}]")
                else:
                    typer.echo(f"Could not find channel info for '{ch_identifier}' in DB or via API. Skipping.")
                    logging.warning(f"Could not find channel info for '{ch_identifier}'. Skipping video download for this channel.")
                    continue

            # Retrieve all videos for this channel from the database
            videos_in_db = c.execute(
                """
                SELECT video_id, title, published_at, duration
                FROM videos
                WHERE channel_id = ? ORDER BY published_at ASC
                """,
                (channel_id,)
            ).fetchall()

            if not videos_in_db:
                typer.echo(f"No videos found in DB for channel {channel_name}. Please run 'fetch' command first.")
                logging.info(f"No videos found in DB for channel {channel_name}.")
                continue

            channel_downloads = 0
            # Filter `videos_in_db` to only include those that don't exist yet on disk.
            videos_to_download_this_run = []
            for video_row in videos_in_db:
                video_id, title, published_at, duration = video_row
                date_prefix = iso8601_to_date(published_at)
                safe_title = sanitize_filename(title)
                parsed_duration = parse_iso8601_duration(duration)
                
                filename_suffix = f"_{actual_preset}" if actual_preset and actual_preset != "best" else ""
                filename = f"{date_prefix}_{safe_title}_{parsed_duration}_{video_id}{filename_suffix}.mp4"
                
                channel_video_folder = os.path.join(VIDEOS_DIR, sanitize_filename(channel_handle))
                filepath = os.path.join(channel_video_folder, filename)

                if not os.path.exists(filepath):
                    videos_to_download_this_run.append(video_row)
            
            total_videos_to_process_for_channel = len(videos_to_download_this_run)
            if total_videos_to_process_for_channel == 0:
                typer.echo(f"  All videos for channel {channel_name} already exist with preset '{actual_preset}'. Skipping downloads for this channel.")
                continue

            current_video_index_for_channel = 0
            for video_row in videos_to_download_this_run:
                current_video_index_for_channel += 1
                total_videos_checked_overall += 1 
                video_id, title, published_at, duration = video_row
                
                typer.echo(f"  Checking video [{current_video_index_for_channel} / {total_videos_to_process_for_channel}]: {title} ({video_id})")
                
                downloaded = download_video(video_id, title, published_at, duration, channel_handle, actual_preset, 
                                            current_video_num=current_video_index_for_channel, 
                                            total_videos_num=total_videos_to_process_for_channel)
                if downloaded: 
                    channel_downloads += 1
                    total_videos_downloaded_overall += 1
            typer.echo(f"  Finished processing channel {channel_name}. Downloaded {channel_downloads} new videos for this channel.")

        typer.echo("\n--- Video Download Summary ---")
        typer.echo(f"Channels processed: {channels_processed_count}")
        typer.echo(f"Total videos checked (potential downloads): {total_videos_checked_overall}")
        typer.echo(f"Total new videos downloaded: {total_videos_downloaded_overall}")
        typer.echo(f"Detailed log available in {os.path.basename(logging.getLogger().handlers[0].baseFilename)}")

    except KeyboardInterrupt:
        typer.echo("\nVideo download operation interrupted by user (CTRL+C). Exiting gracefully.")
        logging.info("Video download operation interrupted by user (CTRL+C).")
        raise typer.Exit(0)
    except Exception as e:
        logging.critical(f"An unhandled error occurred during video download: {e}", exc_info=True)
        typer.echo(f"An unexpected error occurred: {e}. Check log for details.")
        raise typer.Exit(1)
    finally:
        if conn:
            conn.close()
            logging.info("Database connection closed.")

@app.command()
def download(
    video_identifier: str = typer.Argument(..., help="YouTube video URL or ID. Other URL arguments will be ignored."),
    preset: Optional[str] = typer.Option(
        None,
        "--preset",
        "-p",
        help="Download video quality preset (best, normal, small). 'best' is default.",
    ),
    output_dir: Optional[str] = typer.Option(
        None,
        "--dir",
        "-d",
        help="Destination directory for the downloaded video. Defaults to current directory.",
    ),
):
    """
    Downloads a single YouTube video by URL or ID.
    Does not interact with the database.
    """
    valid_presets = ["best", "normal", "small"]
    if preset and preset not in valid_presets:
        typer.echo(f"Error: Invalid preset '{preset}'. Valid options are: {', '.join(valid_presets)}")
        raise typer.Exit(1)
    
    actual_preset = preset if preset else "best"
    
    # Resolve output directory
    resolved_output_dir = os.path.abspath(output_dir) if output_dir else os.getcwd()

    download_video_adhoc(video_identifier, resolved_output_dir, actual_preset)


@app.command(name="test-api-key")
def test_api_key_command(
    endpoint: str = typer.Option(
        "videos",
        "--endpoint",
        "-e",
        help="API endpoint to test (e.g., 'videos', 'playlistItems', 'channels', 'search')."
    ),
    id: str = typer.Option(
        "Ks-_Mh1QhMc",
        "--id",
        "-i",
        help="ID for the resource to test (e.g., video ID, playlist ID, channel ID). For 'search' endpoint, this will be used as the query string."
    )
):
    """
    Tests the YouTube Data API key by making a small request.
    This consumes 1 unit of your API quota.
    """
    typer.echo(f"Testing API key with endpoint '{endpoint}' and ID/query '{id}'...")
    logging.info(f"Testing API key with endpoint '{endpoint}' and ID/query '{id}'.")

    params: Dict[str, Any] = {"part": "id"}
    if endpoint in ["videos", "channels", "playlistItems"]:
        params["id"] = id
    elif endpoint == "search":
        params["q"] = id
        params["type"] = "video"
        params["maxResults"] = 1
    else:
        typer.echo(f"Error: Unknown endpoint '{endpoint}'. Please choose from 'videos', 'playlistItems', 'channels', 'search'.")
        raise typer.Exit(1)

    try:
        response_data = _make_api_request(endpoint, params)
        if response_data and response_data.get("items") is not None:
            typer.echo("API key is valid and request was successful!")
            typer.echo(f"Successfully retrieved {len(response_data['items'])} item(s).")
            logging.info(f"API key test successful for endpoint {endpoint}.")
        else:
            typer.echo("API key seems valid, but no items were returned. Check ID or query.")
            logging.warning(f"API key test returned no items for endpoint {endpoint}.")
    except Exception as e:
        typer.echo(f"API key test failed: {e}")
        logging.error(f"API key test failed for endpoint {endpoint}: {e}")
        raise typer.Exit(1)

@app.command()
def quota():
    """
    Provides a very rough estimate of API quota status by checking against a known endpoint.
    This consumes 1 unit of your daily quota.
    """
    typer.echo("Attempting to fetch API quota status (this consumes 1 unit of quota)...")
    try:
        # A simple request to a public video to check quota without needing specific user data
        response_data = _make_api_request("videos", {"part": "id", "id": "dQw4w9WgXcQ"}) # Rick Astley - Never Gonna Give You Up
        
        # The YouTube API does not expose a direct 'quota remaining' value.
        # We can only infer success/failure based on the response.
        if response_data and response_data.get("items") is not None:
            typer.echo("API quota check successful. This means your key is valid and you have quota available.")
            typer.echo("For exact quota usage, please check your Google Cloud Console for accurate quota status.")
            logging.info("API quota check successful.")
        else:
            typer.echo("API quota check completed, but no items returned. This might indicate quota exhaustion or an issue.")
            typer.echo("Please check your Google Cloud Console for accurate quota status.")
            logging.warning("API quota check completed with no items returned, might indicate issue.")
        
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 403:
            typer.echo(f"Error 403: Forbidden. This often means your API key is invalid or your daily quota has been exceeded.")
            typer.echo("Please check your Google Cloud Console for quota status and API key validity.")
            logging.error(f"API quota check failed with 403: {e}")
        else:
            typer.echo(f"API quota check failed with HTTP error {e.response.status_code}. Check ytextractor.log for details.")
            logging.error(f"API quota check failed with HTTP error: {e}")
        raise typer.Exit(1)
    except Exception as e:
        typer.echo(f"An unexpected error occurred during quota check: {e}. Check ytextractor.log for details.")
        logging.error(f"An unexpected error occurred during quota check: {e}")
        raise typer.Exit(1)


@app.callback(invoke_without_command=True) # Added invoke_without_command=True here
def main(
    version: Optional[bool] = typer.Option(
        None, "--version", "-v", help="Show the application's version and exit."
    ),
):
    """
    A CLI tool for fetching YouTube channel video metadata, thumbnails,
    and downloading videos.
    """
    if version:
        typer.echo(f"ytextractor version: {__version__}")
        raise typer.Exit()
    # If no specific command is called, Typer will show the help message by default.

if __name__ == "__main__":
    app()