DownloadLOFARSpectraData

#!/usr/bin/env python
# coding: utf-8

import requests
from bs4 import BeautifulSoup
from astropy.io import fits
import os
from urllib.parse import urljoin

def download_fits_files(url, output_dir="fits_files"):
    """Download all FITS files from the specified URL directory to output_dir."""
    # Create output directory if it doesn't exist
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    
    try:
        # Fetch the webpage
        response = requests.get(url, timeout=10)
        response.raise_for_status()  # Raise exception for bad status codes
        
        # Parse HTML
        soup = BeautifulSoup(response.text, 'html.parser')
        
        # Find all links ending in .fits
        fits_files = []
        for link in soup.find_all('a'):
            href = link.get('href')
            if href and href.lower().endswith('.fits'):
                fits_files.append(href)
        
        if not fits_files:
            print("No FITS files found in the directory.")
            return
        
        print(f"Found {len(fits_files)} FITS files: {fits_files}")
        
        # Download each FITS file
        for fits_file in fits_files:
            file_url = urljoin(url, fits_file)
            local_path = os.path.join(output_dir, fits_file)
            
            # Skip if file already exists and is valid
            if os.path.exists(local_path):
                try:
                    with fits.open(local_path) as hdu:
                        print(f"Skipping {fits_file}: Already downloaded and valid")
                    continue
                except Exception:
                    print(f"Existing file {fits_file} is corrupted, re-downloading")
            
            print(f"Downloading {fits_file}...")
            try:
                file_response = requests.get(file_url, stream=True, timeout=10)
                file_response.raise_for_status()
                
                # Save the file
                with open(local_path, 'wb') as f:
                    for chunk in file_response.iter_content(chunk_size=8192):
                        if chunk:
                            f.write(chunk)
                
                # Verify FITS file
                try:
                    with fits.open(local_path) as hdu:
                        print(f"Successfully downloaded and verified {fits_file}")
                except Exception as e:
                    print(f"Error: {fits_file} is not a valid FITS file: {e}")
                    os.remove(local_path)  # Remove invalid file
                    continue
                
            except requests.RequestException as e:
                print(f"Failed to download {fits_file}: {e}")
                continue
                
    except requests.RequestException as e:
        print(f"Error accessing URL {url}: {e}")
    except Exception as e:
        print(f"Unexpected error: {e}")

# --- Main Execution ---
url = "https://spaceweather.astron.nl/SolarKSP/data/atdb_process/solar_bf_compressing/12627/2040812/dynamic_spectra/"
output_dir = "lofar_fits_files"  # Directory to save FITS files

download_fits_files(url, output_dir)
print(f"All FITS files downloaded to {output_dir}")