FITS Handling: Introduce Lossless Compression for HDUs

.config

@@ -418,7 +418,10 @@ event_monitor_check_interval_fast: 3
 
 
 [Compression]
-
+; Specify whether the FITS file's data units (HDUs) should be losslessly compressed.
+; When set to 'true', HDUs are losslessly compressed, potentially reducing FITS file size by 30 to 60%.
+; Set to 'false' if compatibility with deprecated FITS software is a concern.
+hdu_compress: true
 
 
 [FireballDetection]

RMS/Compression.py

@@ -153,7 +153,8 @@ def saveFF(self, arr, startTime, N):
             ff.starttime = dt.isoformat(timespec='microseconds')
 
         # Write the FF file
-        FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format)
+        FFfile.write(ff, self.data_dir, filename_millis, fmt=self.config.ff_format,
+                     compress=self.config.hdu_compress)
 
         return filename_millis, filename_micros
 

RMS/ConfigReader.py

@@ -513,8 +513,8 @@ def __init__(self):
         self.event_monitor_check_interval = 30
         self.event_monitor_check_interval_fast = 5
 
-
-
+        ##### Compression
+        self.hdu_compress = False
 
         ##### Weave compilation arguments
         self.extra_compile_args = ["-O3"]
@@ -1410,7 +1410,11 @@ def parseBuildArgs(config, parser):
 
 def parseCompression(config, parser):
     section = "Compression"
-    pass
+    if not parser.has_section(section):
+        return
+
+    if parser.has_option(section, "hdu_compress"):
+        config.hdu_compress = parser.getboolean(section, "hdu_compress")
 
 
 

RMS/Formats/FFfile.py

@@ -99,7 +99,7 @@ def read(directory, filename, fmt=None, array=False, full_filename=False, verbos
 
 
 
-def write(ff, directory, filename, fmt=None):
+def write(ff, directory, filename, fmt=None, compress=True):
     """ Write a FF structure to a FITS file in specified directory.
 
     Arguments:
@@ -140,7 +140,7 @@ def write(ff, directory, filename, fmt=None):
 
     # RMS fits format
     elif fmt == 'fits':
-        writeFFfits(ff, directory, filename)
+        writeFFfits(ff, directory, filename, compress=compress)
 
 
 def reconstructFrame(ff, frame_no, avepixel=False):

RMS/Formats/FFfits.py

@@ -131,7 +131,7 @@ def read(directory, filename, array=False, full_filename=False, memmap=True):
 
 
 
-def write(ff, directory, filename):
+def write(ff, directory, filename, compress=True):
     """ Write a FF structure to a FITS file in specified directory.
 
     Arguments:
@@ -172,12 +172,19 @@ def write(ff, directory, filename):
         ff.avepixel = ff.avepixel[0]
         ff.stdpixel = ff.stdpixel[0]
 
-    # Add the maxpixel to the list
-    maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL')
-    maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME')
-    avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL')
-    stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL')
+    # Add the data arrays to the list and specify compression algorithm
+    if compress:
+        maxpixel_hdu = fits.CompImageHDU(ff.maxpixel, name='MAXPIXEL', compression_type='RICE_1')
+        maxframe_hdu = fits.CompImageHDU(ff.maxframe, name='MAXFRAME', compression_type='RICE_1')
+        avepixel_hdu = fits.CompImageHDU(ff.avepixel, name='AVEPIXEL', compression_type='RICE_1')
+        stdpixel_hdu = fits.CompImageHDU(ff.stdpixel, name='STDPIXEL', compression_type='RICE_1')
 
+    else:
+        maxpixel_hdu = fits.ImageHDU(ff.maxpixel, name='MAXPIXEL')
+        maxframe_hdu = fits.ImageHDU(ff.maxframe, name='MAXFRAME')
+        avepixel_hdu = fits.ImageHDU(ff.avepixel, name='AVEPIXEL')
+        stdpixel_hdu = fits.ImageHDU(ff.stdpixel, name='STDPIXEL')
+
     # Create the primary part
     prim = fits.PrimaryHDU(header=head)
 

Utils/ConvertCompressedFits.py

@@ -0,0 +1,110 @@
+"""
+This script converts FITS files between compressed and uncompressed formats.
+Designed to work recursively, it searches for all .fits files within a specified input directory
+and its subdirectories, then converts each file preserving the original directory structure
+in a separate output directory.
+
+Usage:
+    python ConvertCompressedFits.py <input_directory> <output_directory> [--compress | --decompress]
+
+Where:
+    <input_directory> is the directory containing the .fits files to convert.
+    <output_directory> is the directory where the converted .fits files will be saved.
+    --compress: Convert uncompressed FITS to compressed (RICE_1) format.
+    --decompress: Convert compressed FITS to uncompressed format (default).
+
+Examples:
+    python ConvertCompressedFits.py /path/to/uncompressed /path/to/compressed --compress
+    python ConvertCompressedFits.py /path/to/compressed /path/to/uncompressed --decompress
+
+"""
+
+import argparse
+import os
+import sys
+import glob
+
+from RMS.Formats.FFfits import read as readFFfits
+from RMS.Formats.FFfits import write as writeFFfits
+
+
+def findFitsFiles(directory):
+    """Recursively find all FITS files in a directory and its subdirectories.
+
+    Arguments:
+        directory: [str] The path to the directory to search.
+
+    Return:
+        [list] List of paths to FITS files found.
+
+    """
+    fits_files = []
+    for root, dirs, files in os.walk(directory):
+        fits_files.extend(glob.glob(os.path.join(root, '*.fits')))
+    return fits_files
+
+
+def convertDirectoryFits(input_directory, output_directory, compress=False):
+    """Convert all FITS files found in the input directory.
+
+    Arguments:
+        input_directory: [str] The path to the folder containing the files to convert.
+        output_directory: [str] The path to the folder where the converted files will be saved.
+        compress: [bool] If True, compress the output files. If False, decompress them.
+
+    Return:
+        None
+
+    """
+    fits_files = findFitsFiles(input_directory)
+    action = "Compressed" if compress else "Decompressed"
+
+    for fits_file in fits_files:
+        try:
+            # Preserve the path structure in the output directory
+            relative_path = os.path.relpath(fits_file, input_directory)
+            output_path = os.path.join(output_directory, relative_path)
+            os.makedirs(os.path.dirname(output_path), exist_ok=True)
+
+            # Read the FITS file
+            ff = readFFfits('', fits_file, array=True, full_filename=True)
+
+            # Write with specified compression setting
+            writeFFfits(ff, os.path.dirname(output_path), os.path.basename(output_path), compress=compress)
+            print(f"{action} {fits_file} to {output_path}")
+        except Exception as e:
+            print(f"Failed to convert {fits_file}: {e}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+
+    ### COMMAND LINE ARGUMENTS
+
+    # Init the command line arguments parser
+
+    arg_parser = argparse.ArgumentParser(description="Convert FITS files between compressed and uncompressed "
+                                         "formats in a directory (and subdirectories).")
+
+    arg_parser.add_argument("input_directory",
+                            help="The directory to search for FITS files.")
+
+    arg_parser.add_argument("output_directory",
+                            help="The directory where the converted FITS files will be saved.")
+
+    mode_group = arg_parser.add_mutually_exclusive_group()
+    mode_group.add_argument("--compress", action="store_true",
+                            help="Compress FITS files using RICE_1 algorithm.")
+    mode_group.add_argument("--decompress", action="store_true", default=True,
+                            help="Decompress FITS files (default).")
+
+    cml_args = arg_parser.parse_args()
+
+    #########################
+
+    if not os.path.isdir(cml_args.input_directory):
+        print("The specified input directory does not exist.", file=sys.stderr)
+        sys.exit(1)
+
+    os.makedirs(cml_args.output_directory, exist_ok=True)
+
+    convertDirectoryFits(cml_args.input_directory, cml_args.output_directory, compress=cml_args.compress)