decode.cpp

/*
  This program decodes CRFS BIN files and translate than into HDF5 format performing carrier detection on the process. Detailed background information can be obtained at the TFM from Fábio Santos Lobão at mugi.upv.es. Coded by Fábio Santos Lobão, falosan@inf.upv.es

  Program arguments arguments are the following:
    "-d" to select a decoding mode. Optional, default is zero.
    "-u" to select the output measurement unit to be reported. Optional, default is 0.
    "-f" preceeding the filename for the input BIN file.
    "-o" preceeding the destination file or folder where the data will be stored.
         If a file already exists with the indicated filename, it will be overwritten
         If only a path to a folder is provided, a file will be created with name based on the timestamp when the data was collected and its coordinates, with extension ".h5"
    "-h" for simple help.
    
    The order in which the options are presented is not relevant, but if a parameter is needed for the option it should be the following string without blanc spaces
      -h option have precedence and will null other options

    examples: 
      ./decode -d 0 -u 0 -f ./InBox/SCAN_M_450470_rfeye002088_170426_235831.bin -o ./OutBox
      ./decode -d 1 -u 1 -f ./InBox/SCAN_M_450470_rfeye002088_170426_235831.bin -o ./OutBox/ouput.h5

  If the file is a CRFS bin file type 22, generated by RFEye Logger, it will be decoded and,
      if successfull, will generate files containing RF spectrum channel information and associated occupancy related statistics.
      Empty channels and timeslots will be recorded as noise statistics 

  This was coded using visual studio code within linux, using gcc libraries and c11 standard for higher iteroperability with other systems
  HDF5 library is also required
  Use this addon "Better Comments" extension to improved readability
*/

// External Libraries
  #include <H5Cpp.h>
  #include <limits.h>
  #include <iostream>
  #include <fstream>
  #include <string>
  #include <sstream>
  #include <ctime>
  #include <vector>
  #include <math.h>
  #include <unordered_map>
  #include <map>
  #include <sys/resource.h>
  #include <sys/types.h> 
  #include <sys/stat.h> 
  
// used only for debug with "level_differential_detector.h" 
  #include "gnuplot-iostream.h"

// Project Libraries
  #include <crfsbin.hpp>    // constants associated with the CRFS bin file format
  #include <h5_spectrum.hpp> // constants associated with the created HDF5 standard for spectrum data
  #include "level_differential_detector.hpp" // object to perform channel detection based on the signal level differential.
  
// Namespaces
  using namespace std;
  using namespace H5;

// Constants used by program workflow
  const char PARAMETER_SWITCH = '-';  
  const char HELP_OPTION = 'h'; 
  const char DETECTION_MODE_OPTION = 'd';
  const int NO_DETECTION_OPTION = 0;
  const int LEVEL_DETECTION_OPTION = 1;
  const int FIRST_DETECTION_MODE_OPTION = NO_DETECTION_OPTION;
  const int LAST_DETECTION_MODE_OPTION = LEVEL_DETECTION_OPTION;
  const char OUTPUT_UNIT_OPTION = 'u';
  const int DBM = 0; // corresponds to dBm
  const int DBUV_M = 1; // corresponds to dBuV/m
  const double DBM_TO_DBUV_FACTOR = 107.0; // transformation factor from dBm to dBuV for 50ohms impedance.
  const char FILENAME_OPTION = 'f';
  const char OUTPUT_DESTINATION_OPTION = 'o';

// Constant that control de behaviour of the program on specific conditions 
  // Maximum number of levels for a CRFS bin file. Original format has a 0.5dB resolution and a range of 127dB, making up 255 bin levels. Increasing the resolution to 0.25dB to reduce error
    const uint32_t MAXIMUM_NUMBER_OF_LEVEL_BINS = 256;
  // Reference to the byte definition
    const double  BYTE_MAX_VALUE = 255.0;
  // Default level step computed from bin standard definition and maximum number of level bins
    const double  DEFAULT_LEVEL_STEP = ( BYTE_MAX_VALUE / STEP_PER_LEVEL_UNIT ) / (MAXIMUM_NUMBER_OF_LEVEL_BINS-1);
  // Reference to the conversion from nanosencod into second
    const double  NANOSECONDS_IN_SECOND = 1000000000.0;
  // Reference to be used to mark segments of the spectrum that where removed. 
    const float   NULL_SPECTRUM = -99999;
  // Minimum usable ID for channel stream information. Also the limit for CRFS thread ID that can be converted to stream ID 
    const double  MINIMUM_CHANNEL_THREAD_ID = 20000;
  // responsible
    const string  RESPONSIBLE("falosan@inf.upv.es");

// Global variables used for application control
  // store the different block types
    int       block_type_count[MAXIMUM_BLOCK_NUMBER+1];
  // allow for different detection mode
    char      detection_mode = NO_DETECTION_OPTION;
  // store the input parameter related to the requested output measurement unit
    string    output_unit_label;
    char      output_unit = DBM;
  // point to byte being processed from the data loaded
    char     *memblock;
  // store the amount of data stored on the file, used for estimating memory usage and perform allocation
    streampos data_size;
  // string storage for JSON output
    stringstream output_log;

// Object used to store parameters used for data conversion and detection
struct sweep_parameters {
      int       window_size;
      int       second_window_offset;
      double    detection_threshold;
    // in number of bins, no channel will be smaller than this size
      int       minimum_channel_width; 
    // in number of bins, no channel will be spaced with less than this size
      int       minimum_ascending_break_separation;
      int       minimum_descending_break_separation;
    // in number of bins, no channel will be larger than this size. A break will be automatically created
      int       maximum_channel_width;
    // size of the channel size array
      int       number_frequency_bins;
    // pointer to 1D array that list the trace data streams over all frequency bins, allowing for fast adjustment
      double  * level_offset_factor_frequency;  
    // antenna factor used for the adjustment to an isotropic antenna equivalent
      double  * level_offset_factor;
    // level offset factor minimum value
      double    level_offset_factor_min;
    // level offset factor maximum value
      double    level_offset_factor_max;

  // Null constructor
    sweep_parameters() : window_size(5), 
                         second_window_offset(1),
                         detection_threshold(7.0),
                         minimum_channel_width(15),
                         minimum_ascending_break_separation(2),
                         minimum_descending_break_separation(2),
                         maximum_channel_width(125),
                         number_frequency_bins(0),
                         level_offset_factor_frequency(nullptr),
                         level_offset_factor(nullptr),
                         level_offset_factor_min(__DBL_MAX__),
                         level_offset_factor_max(-__DBL_MAX__) {}

  // constructor to an specific frequency range
    sweep_parameters (double start_frequency,
                      double stop_frequency,
                      uint32_t number_frequency_bins_data,
                      int input_unit,
                      int output_unit,
                      string antenna_id) {

      // TODO: Load the data to these variables from the database, add database handle to input parameters
        double differential_threshold_detection_level = 7.0; // in dB
        double frequency_offset_for_differential_computation = 3125.0; // in Hz
        double window_size_for_differential_computation = 6250.0; // in Hz - Actual channel spacing should be more than this value to ensure detection
        double minimum_channel_width_reference = 6250.0; // in Hz, no channel will be smaller than this size
        double minimum_channel_spacing_at_start = 781; // in Hz, no channel will be spaced with less than this size
        double minimum_channel_spacing_at_end = 781; // in Hz, no channel will be spaced with less than this size
        double maximum_channel_width_reference = 1000000; // in Hz, no channel will be larger than this size. A break will be automatically created
        
      // load the antenna factor data 
      // TODO: the following variable initialization should be performed with data loaded from the database
        int antenna_factor_length = 17;
        double antenna_factor_data[2][17] = { {  20000000,   30000000,  40000000,  50000000,  80000000,  100000000,  150000000,  200000000,  300000000,  400000000,  500000000,  600000000,  700000000,  800000000,  900000000, 1000000000, 1200000000},					
                                              {      18.8,       12.8,       9.3,       7.7,       8.1,       10.3,       17.3,       21.1,       25.8,       27.3,       27.2,       28.8,       31.9,       31.0,       30.3,       33.7,       31.3} };

      // initialize variables used to store the reference to the correction factor
        int level_offset_factor_length = 2;
        double *level_offset_factor_data;

        if (  input_unit == output_unit ) {
            level_offset_factor_length = 2;
            level_offset_factor_data = new double[2*level_offset_factor_length];
            level_offset_factor_data[0] = start_frequency;
            level_offset_factor_data[1] = stop_frequency;					
            level_offset_factor_data[2] = -STANDARD_LEVEL_OFFSET;
            level_offset_factor_data[3] = -STANDARD_LEVEL_OFFSET;
        }
        else {
          // no detection simple store the data for later processing
            switch ( (output_unit*10) + input_unit )   {
              case DBUV_M*10+DBM : {
                // locate the initial position of the two references to be used for the first interpolation (finish pointing to the higher frequency boundary)
                  double  * antenna_factor_iterator = &antenna_factor_data[0][0];
                  double  * pointer_end = &antenna_factor_data[0][antenna_factor_length];
                  bool      inside_array = true; // using this variable is possible to skip the second loop in case the first reaches the end of the array
                  while ( (*antenna_factor_iterator < start_frequency) && inside_array ) {
                    antenna_factor_iterator++;
                    if ( antenna_factor_iterator >= pointer_end ) inside_array = false;
                  }

                // locate the final position of the two references to be used for the first interpolation and update the counter associated with the number of factors to be used (finish pointing to the higher frequency boundary)
                  while ( (*antenna_factor_iterator < stop_frequency) && inside_array ) {
                    antenna_factor_iterator++;
                    level_offset_factor_length++;
                    if ( antenna_factor_iterator >= pointer_end ) inside_array = false;
                  }

                // create the array that will be used for later processing and store on it the corresponding values
                  level_offset_factor_data = new double[2*level_offset_factor_length];
                  for ( int i = 0; i < level_offset_factor_length; i++ ) {
                    level_offset_factor_data[i] = antenna_factor_iterator[i-level_offset_factor_length+1];
                    level_offset_factor_data[i+level_offset_factor_length] = antenna_factor_iterator[(i-level_offset_factor_length+1)+antenna_factor_length]-STANDARD_LEVEL_OFFSET+DBM_TO_DBUV_FACTOR;
                  }
              } break;
              default :
                // output message 
                  output_log << "\n\t\t\"Warning\": \"No conversion method between input measurement unit type " << input_unit << " and output measurement unit type " << output_unit << "\",";
            }
        }

      // compute bin size in Hz
        double bin_size = (stop_frequency - start_frequency)/((double)number_frequency_bins_data-(double)1.0);      
      
      // initialize variables to the object
        window_size = (int)round(window_size_for_differential_computation/bin_size);
        second_window_offset = (int)round(frequency_offset_for_differential_computation/bin_size);
        detection_threshold = differential_threshold_detection_level;
        minimum_channel_width = (int)round(minimum_channel_width_reference/bin_size);
        minimum_ascending_break_separation = (int)round((2.0*minimum_channel_spacing_at_start)/bin_size);
        minimum_descending_break_separation = (int)round((2.0*minimum_channel_spacing_at_end)/bin_size);
        maximum_channel_width = (int)round(maximum_channel_width_reference/bin_size);

        number_frequency_bins = number_frequency_bins_data;

      // compute the frequency step between bins center frequency
        double frequency_step = (stop_frequency - start_frequency)/((double)number_frequency_bins-(double)1.0);      

      // initialize vectors for the channel stream index        
        level_offset_factor_frequency = new double[number_frequency_bins];
        level_offset_factor_frequency[0] = start_frequency; 

      // initialize the frequency vector and end condition variable
        double * level_offset_factor_frequency_iterator = level_offset_factor_frequency;

      // initialize values for the range of the level offset
        level_offset_factor_min = __DBL_MAX__;
        level_offset_factor_max = -__DBL_MAX__;

      // initialize vectors for the level offset ( antenna factor +  correction )
        level_offset_factor = new double[number_frequency_bins];
        double * level_offset_factor_iterator = level_offset_factor;

      // declare variables used to compute interpolation coefficients
        double factor_gradient = 0.0;
        double factor_offset = 0.0;
        double delta_frequency = 0.0;
        double next_factor_frequency = 0.0;
        double frequency_exact_value = start_frequency;

      // define lambda function compute interpolation coefficients
        auto compute_coefficient = [&] {
          factor_gradient = (level_offset_factor_data[level_offset_factor_length]-level_offset_factor_data[level_offset_factor_length+1]) / (level_offset_factor_data[0]-level_offset_factor_data[1]);
          factor_offset = level_offset_factor_data[level_offset_factor_length];
          delta_frequency = start_frequency - level_offset_factor_data[0];
          next_factor_frequency = level_offset_factor_data[1];
        };

      // define lambda function to compute the coeffiecient variation range
        auto update_range = [&] {
          if ( level_offset_factor_min > *level_offset_factor_iterator ) level_offset_factor_min = *level_offset_factor_iterator;
          if ( level_offset_factor_max < *level_offset_factor_iterator ) level_offset_factor_max = *level_offset_factor_iterator;
        };

      // compute initial interpolation coefficients
        compute_coefficient();            

      // sweep the frequency array, filling the corresponding antenna factor and antenna frequency arrays
        double * pointer_end = &level_offset_factor_frequency[number_frequency_bins];
        do {
          // compute frequency value for the next position at the array, rounding
            *level_offset_factor_frequency_iterator = round(frequency_exact_value);

          // check if antenna factor still aplies to the current frequency
            if ( *level_offset_factor_frequency_iterator > next_factor_frequency  ) {
              // if not, update linear interpolation coefficients 
              level_offset_factor_data++;
              compute_coefficient();
            }

          // compute antenna factor for the current position
            *level_offset_factor_iterator = (factor_gradient*delta_frequency) + factor_offset;

          // update the level range information
            update_range();

          // increment pointer for the next loop
            level_offset_factor_frequency_iterator++;
            level_offset_factor_iterator++;
            delta_frequency+=frequency_step;
            frequency_exact_value+=frequency_step;

        } while ( level_offset_factor_frequency_iterator < pointer_end );
    }
};

// structure to store and perform online computation of basic descriptive indexes for a normally distributed variable   
struct normal {
  // data fields
    double mean_value; // mean_value = ((count*mean_value)+ X )/(count+1); 
    double std_value; // std_value = ( n-2 / n-1 ) std_value {n-1}+{1\over n}(X_n-\bar X_{n-1})^2.
    int    count; // count = count + 1;
    double sum; // to reduce the computational effort and rounding error on the average computation
    double sum_squares; // to reduce the computational effort and reduce error on the standard deviation computation

  // null argument constructor for this structure
    normal() : mean_value(0.0), std_value(0.0), count(0), sum(0.0), sum_squares(0.0) {}

  // Perform online computation to add an element to the object
    /*
    @ARTICLE{Welford62noteon,
      author = {Author(s) B. P. Welford and B. P. Welford},
      title = {Note on a method for calculating corrected sums of squares and products},
      journal = {Technometrics},
      year = {1962},
      pages = {419--420}  
    }
    */

  // add element to the standard normal distribution
    void add_element (double new_element) {
      // local variable to help on the computation
        double previous_mean_value;
        double delta;
      
      // select appropriate update procedure according to the number of elements. 
      // for better efficiency, considering a vector with 3 or more elements, first test this general and otherwise move to the other cases
        if( count > 1 ) {
            previous_mean_value = mean_value;
            
            sum = sum + new_element; 
            count++;
            mean_value = sum / (double)count;
            
            sum_squares = sum_squares + ((new_element-previous_mean_value)*(new_element-mean_value));

            std_value = sqrt ( sum_squares / (double)count );
        } else {
          if ( count < 1 ) {
            // if there are 0 (negative number of elements are considered 0), set as the first element
              mean_value = new_element;
              count = 1;
              sum = new_element;
          } else {
              count = 2;
              mean_value = (mean_value + new_element) / (double)count;
              sum = sum + new_element;
              delta = new_element-mean_value;
              sum_squares = delta*delta;
          }
        }
    }
};

// structure to store and perform online computation of basic descriptive indexes for a variable not normally distributed   
struct non_normal {
  // data fields
    float maximum_value;
    float minimum_value;
    vector<float> data;

  // null argument constructor for this structure
    non_normal() : maximum_value(0.0), minimum_value(0.0), data() {}

  // Perform online computation to add an element to the object
    void add_element (double new_element) {

      // store the element on the vector
        data.push_back(new_element);

      // compute maximum, minimum
        if (new_element>maximum_value) {
          maximum_value = new_element;
        } else {
          if (new_element<minimum_value) {
            minimum_value = new_element;
          }
        }
    }
};

// structures to store recovered positional data
struct position {
  // data fields
    struct timespec start_clock;
    struct timespec stop_clock;
    struct non_normal satellite_fix;
    struct normal latitude;
    struct normal longitude;
    struct normal altitude;
    struct normal horizontal_variation;

  // null argument constructor for this structure
    position() : start_clock(), stop_clock(), satellite_fix(), latitude(), longitude(), altitude(), horizontal_variation() {}

  // Update time references to the structure
    void add_time (timespec new_time) {

      // If a time was already recorded, store the new time as the stop clock, otherwise set it as the start
        if (start_clock.tv_sec>0) {
          stop_clock = new_time;
        } else {
          start_clock = new_time;
        }
    }
};

// structure to store runtime events
struct event_item {
  // data fields
    uint32_t posix_time;
    uint32_t nanosecond_time; 
    string type;
    string value;
  
  // null argument constructor for this structure
    event_item() : posix_time(), nanosecond_time(), type(), value() {}

  // full object constructor
  event_item (struct timespec time_data, 
              string type_data,
              string value_data) {

    posix_time = (uint32_t)time_data.tv_sec;
    nanosecond_time = (uint32_t)time_data.tv_nsec;
    type = type_data;
    value = value_data;
  }

  // operator to perfor vector sorting by the posix timestamp
    bool operator < (const event_item& str) const
      {
          return (posix_time < str.posix_time);
      }
};

// structures to store recovered equipment data
struct equipment_data {
  // data fields
    string hostname;
    string unit_info;
    string method;

  // null argument constructor for this structure
    equipment_data() : hostname(""), unit_info(""), method("") {}

  // Register equipment information as part of the measurement log
    void set_to_log(
                    // Initial measurement timestamp, that will be used for reference on the log
                      struct timespec time_data,
                    // Log to store the information
                      vector<event_item> *event_log_pointer) {

      event_log_pointer->push_back(event_item(time_data,CRFS_HOSTNAME,hostname));
      event_log_pointer->push_back(event_item(time_data,CRFS_UNIT_INFO,unit_info));
      event_log_pointer->push_back(event_item(time_data,METHOD,method));
    }

};

// Constants to be used to create a tristate simplified logic using int
  const int UNKNOWN = 0;
  const int BUSY = 1;
  const int FREE = -1;

// structure that stores a channel status event
  struct channel_status {
    // data fields
      // Channel status is "1" for occupied (with detected emission within the channel core), "-1" for unoccupied or "0" for unknown.
        int         status;          
      // posix time of the event occurrence
        uint32_t    posix_time; 
      // nanosecond added time of the event occurrence              
        uint32_t    nanosecond_time;
      // duration of the status
        double      duration;

    // null argument constructor for this structure
      channel_status() :
        status(UNKNOWN),
        posix_time(0),
        nanosecond_time(0),
        duration(0.0) {}       

    // method to set all values.
      void set(
        // Channel status is "true" for occupied (with detected emission within the channel core) or false otherwise
          int         status_data,          
        // posix time of the event occurrence
          uint32_t    posix_time_data, 
        // nanosecond added time of the event occurrence              
          uint32_t    nanosecond_time_data) {

          status = status_data;
          posix_time = (uint32_t)posix_time_data;
          nanosecond_time =(uint32_t)nanosecond_time_data;
      }

    // method to convert from three state to bool will return error if status unknown
      bool is_active(void) {

        switch ( status ) {
          case BUSY :
              return true;
            break;
          case FREE :
              return false;
            break;
          default :
            return EXIT_FAILURE;
        }

      }

  };

// structure to store trace data relative to one capture 
struct channel_data {
  // data fields
    // Stream code based on central frequency in kHz, rounded to 1kHz
      double               stream_code;
    // Initial frequency of the channel
      double               start_frequency;
    // Final frequency of the channel
      double               stop_frequency;
    // number of frequency bins on the power signature
      uint32_t             number_frequency_bins;
    // frequency axis
      double              *frequency_axis;
    // number of level bins on the power signature    
      uint32_t             number_level_bins;
    // step between two level bins
      double               level_step;
    // level axis values
      double              *level_axis;
    // 2D array power frequency relation for all detected occurrences 
      uint16_t            *power_signature;
    // Number of traces considered
      uint32_t             number_of_samples;
    // Level normal approximation statistics
      vector<normal>       level_normal;
    // Level density inside the core over time
      vector<float>        core_level_density;
    // Posix time axis for the core density vector
      vector<uint32_t>     posix_time_for_density;
    // Nanosecond time axis for the core density vector
      vector<uint32_t>     nanosecond_time_for_density;
    // POSIX time of the start of the on event
      vector<uint32_t>     ON_event_Posix;
    // uint32_t t f the start of the on event
      vector<uint32_t>     ON_event_Nanosecond;
    // uint32_ton  the on_event
      vector<double>       ON_event_duration;
    // POSIX time of the start of the on event
      vector<uint32_t>     OFF_event_Posix;
    // uint32_time of the start of the on event
      vector<uint32_t>     OFF_event_Nanosecond;
    // uint32_ton  the on_event
      vector<double>       OFF_event_duration;
    // Beginning Status. Used for later file merging 
      channel_status       begin_status;
    // Initial Status Duration. Used for later file merging
      channel_status       end_status;
    // Average sample rate
      normal               sample_rate_period;
    // initial frequency of the channel core
      double               core_initial_frequency;
    // final frequency of the channel core
      double               core_final_frequency;

  // null argument constructor for this structure
    channel_data() :
      stream_code(0.0),
      start_frequency(0.0),
      stop_frequency(0.0),
      number_frequency_bins(0),
      frequency_axis(nullptr),
      number_level_bins(0),
      level_step(0.0),
      level_axis(nullptr),
      power_signature(nullptr),
      number_of_samples(0),
      level_normal(),
      core_level_density(),
      posix_time_for_density(),
      nanosecond_time_for_density(),
      ON_event_Posix(),
      ON_event_Nanosecond(),
      ON_event_duration(),
      OFF_event_Posix(),
      OFF_event_Nanosecond(),
      OFF_event_duration(),
      begin_status(),
      end_status(),
      sample_rate_period(),
      core_initial_frequency(0.0),
      core_final_frequency(0.0) {}

  // Constructor to an specific volume of spectrum data. Allocate memory space.
    channel_data(
      // Lower channel frequency edge
        double               start_frequency_data,
      // Higher channel frequency edge
        double               stop_frequency_data,
      // number of frequency bins on the power signature
        uint32_t             number_frequency_bins_data,
      //
        double               level_step_data,
      // minimum level value
        double               min_level,
      // maximum level value
        double               max_level) {
      
      // compute the initial estimate for the stream code
        stream_code = round((stop_frequency_data+start_frequency_data)/2000.0);

      // save data that was passed and allocate memory to arrays
        start_frequency = start_frequency_data;
        stop_frequency = stop_frequency_data;

        number_frequency_bins = number_frequency_bins_data;

        level_step = level_step_data;

        number_level_bins = (uint32_t)round((max_level-min_level)/level_step)+1;

        try {
          frequency_axis = new double[number_frequency_bins_data];
        }
        catch (bad_alloc& ba) {
          cerr << "Channel frequency axis bad_alloc caught: " << ba.what() << '\n';
        }
        try {
          level_axis = new double[number_level_bins];
        }
        catch (bad_alloc& ba) {
          cerr << "Channel level axis bad_alloc caught: " << ba.what() << '\n';
        }
        try {
          power_signature = new uint16_t[number_level_bins*number_frequency_bins_data]();
        }
        catch (bad_alloc& ba) {
          cerr << "Channel power signature bad_alloc caught: " << ba.what() << '\n';
        }

        level_normal.resize(number_frequency_bins_data);

        number_of_samples = 0;

        begin_status = channel_status();
        end_status = channel_status();
        sample_rate_period = normal();

      // fill the level index array
        double * level_axis_iterator = level_axis;
        double * level_axis_end = &level_axis[number_level_bins];

        *level_axis_iterator = max_level;
        level_axis_iterator++;
        while ( level_axis_iterator < level_axis_end ) {
          *level_axis_iterator = level_axis_iterator[-1]-level_step;
          level_axis_iterator++;
        }
    }

  // operator to perfor vector sorting
    bool operator < (const channel_data& str) const
      {
          return (start_frequency < str.start_frequency);
      }

  // shrink arrays to save memory
    void shrink_to_fit(
            // 
              uint32_t minimum_level_index,
            // trace that contains the segment to be moved
              uint32_t maximum_level_index) {
      // TODO: Create a method to reduce memory usage by reducing the arrays to fit the data exactly

              }

};

// structure to store trace data relative to one capture 
struct trace_data {
  // data fields
    // number of measurement loops performed to create this trace
      uint32_t                number_of_loops;
    // wideband sample duration time using time time in nanoseconds for direct compatibility with time.h
      double                  sample_duration;
    // size of the frequency vector
      uint32_t                number_frequency_bins;
    // size of the time vector
      uint32_t                number_time_bins;
    // 1D array with length number_frequency_bins and that store the frequency axis or indexes to channel streams
      double                 *frequency;
    // frequency space between two adjascent bins 
      double                  frequency_step;
    // 1D array with length number_time_bins with posix time with no leap second 
      uint32_t               *posix_time;
    // 1D array with length number_time_bins with nanosecond precision information to be added to posix time.
      uint32_t               *nanosecond_time;
    // 2D array with dimensions of number_frequency_bins x number_time_bins with power level for each time and frequency bin
      float                  *spectrogram;
    // 2D array with dimensions of number_frequency_bins x number_time_bins with power level for each time and frequency bin
      double                 *spectrogram_sum;
    // 2D vector with list of the index of channels detected on each trace
      vector<detection_flag>  threshold_crossing;
    // 1D vector with noise reference level computed for each trace
      vector<double>          noise_reference;
    // Applied threshold above noise
      double                  relative_thershold_value;
    // Minimum expected level value using full scale by default. Adjusted to the minimum observed level if channel detection is performed.
      double                  level_min;
    // Maximum expected level value using full scale by default. Adjusted to the minimum observed level if channel detection is performed.
      double                  level_max;
    // Index on the level axis to the minimum level observed 
      uint32_t                level_min_index;
    // Index on the level axis to the maximum level observed 
      uint32_t                level_max_index;
    // Channel code, index to the corresponding channel information, if available
      double                  channel_code;

  // null argument constructor for this structure
    trace_data() :
      number_of_loops(0),
      sample_duration(0),
      number_frequency_bins(0),
      number_time_bins(0),
      frequency(nullptr),
      frequency_step(0.0),
      posix_time(nullptr),
      nanosecond_time(nullptr),
      spectrogram(nullptr),
      spectrogram_sum(nullptr),
      threshold_crossing(),
      noise_reference(),
      level_min (__DBL_MAX__),
      level_min_index(0),
      level_max (-__DBL_MAX__),
      level_max_index(UINT32_MAX),
      channel_code(0) {}

  // Constructor to an specific volume of spectrum data. Allocate memory space.
    trace_data(
        uint32_t number_of_loops_data,
        uint32_t sample_duration_data,
        uint32_t number_frequency_bins_data,
        int      estimated_number_time_bins,
        double   start_frequency_data,
        double   stop_frequency_data,
        timespec time_data,
        bool     flag_store_event)      {
      
      // save data that was passed
        number_of_loops = number_of_loops_data;
        
        sample_duration = sample_duration_data;
        
        number_frequency_bins = number_frequency_bins_data;

      // allocate memory to store an array with frequency data
        // frequency = &vector<double>(number_frequency_bins)[0];// (double*) malloc (number_frequency_bins);
        frequency = new double[number_frequency_bins];

      // compute the step of each bin
        frequency_step = (stop_frequency_data - start_frequency_data)/((double)number_frequency_bins-(double)1.0);
        double frequency_exact_value = start_frequency_data;
      
      // create array with frequencies values to all measurement bins  
        double * pointer_end = &frequency[number_frequency_bins];
        double * frequency_iterator = &frequency[1];
        frequency[0] = start_frequency_data;
        while (frequency_iterator < pointer_end)
        {
          frequency_exact_value+=frequency_step;
          *frequency_iterator = round(frequency_exact_value);
          frequency_iterator++;
        }

      // allocate memory to store the array with time data
        posix_time = new uint32_t[estimated_number_time_bins];
        nanosecond_time = new uint32_t[estimated_number_time_bins];

      // store the time information for the first trace
        posix_time[0] = (uint32_t)time_data.tv_sec;
        nanosecond_time[0] = (uint32_t)time_data.tv_nsec;

        number_time_bins = 0;

        level_min = __DBL_MAX__;
        level_min_index = UINT32_MAX;
        level_max = -__DBL_MAX__;
        level_max_index = 0;

        int number_of_elements = number_frequency_bins*(estimated_number_time_bins+1);

      // allocate memory to store the array with spectrogram
        try {
          spectrogram = new float[number_of_elements];
        }
        catch (bad_alloc& ba) {
          cerr << "spectrogram bad_alloc caught: " << ba.what() << '\n';
        }
        try {        
          spectrogram_sum = new double[number_of_elements];
        }
        catch (bad_alloc& ba) {
          cerr << "spectrogram bad_alloc caught: " << ba.what() << '\n';
        }
    }

  // Method to test if a trace copy is possible
    bool test_copy(
            // Trace data object that will receive the moved data at the end postion
              trace_data *destination_trace_data,
            // trace that contains the segment to be moved
              uint32_t    trace_time_index,
            // Initial trace data position
              uint32_t    trace_frequency_begin_index,
            // Number of bins to be moved
              uint32_t    trace_frequency_end_index) {
    
      // Test if the number of frequency bins at the destination is compatible with the number of bins to be copied
      // TODO: Improve error handling on this case and break execution

        if ( destination_trace_data->number_frequency_bins != (trace_frequency_end_index - trace_frequency_begin_index)  ) {
          // report error
            return false;
        } 
        // if the frequency array has the same size, check if the initial value is NOT the same
        else {
          // if the initial frequency value for source and destination are different
            if ( *destination_trace_data->frequency != frequency[trace_frequency_begin_index] ) {
              // report error
                return false;
            }
          // if the initial value is the same, check if the last is also the same
            else if ( destination_trace_data->frequency[destination_trace_data->number_frequency_bins-1] == frequency[trace_frequency_end_index] ){
              // report error              
                return false;
            }
          // else, the frequency array at the destination matches the frequency array at the source and the move may proceed
            else {
              return true;
            }
        }
    }

  // Method to copy segment of a trace into another and delete the original trace data, updating channel information in the process
    void move_channel_trace(
            // Trace data object that will receive the moved data at the end postion
              trace_data *destination_trace_data,
            // Channel data object that will receive the moved data at the end postion
              channel_data *destination_channel_data,
            // trace that contains the segment to be moved
              uint32_t    trace_time_index,
            // Initial trace data position
              uint32_t    trace_frequency_begin_index,
            // Number of bins to be moved
              uint32_t    trace_frequency_end_index) {
    
      // number of measurement loops performed to create this trace
        destination_trace_data->number_of_loops = number_of_loops;
      // wideband sample duration time using time time in nanoseconds for direct compatibility with time.h
        destination_trace_data->sample_duration = sample_duration;

      // Insert new value at the end of the time arrays 
        destination_trace_data->posix_time[destination_trace_data->number_time_bins] = posix_time[trace_time_index];
        destination_trace_data->nanosecond_time[destination_trace_data->number_time_bins] = nanosecond_time[trace_time_index];

      // Copy the spectrogram values and delete the original, computing minimum and maximum values during this process
        float *source_iterator = &spectrogram[(trace_time_index*number_frequency_bins)+trace_frequency_begin_index];
        float *source_end_pointer = &spectrogram[(trace_time_index*number_frequency_bins)+trace_frequency_end_index+1];
        float *target_iterator = &destination_trace_data->spectrogram[destination_trace_data->number_time_bins*destination_trace_data->number_frequency_bins];

      // reset frequency index to point at the first element on the vector
        uint32_t frequency_index = 0;

      // compute a reference index based on the first level and reset maximum and minimum values
        uint32_t level_index = (int)round((level_max-(double)*source_iterator)/destination_channel_data->level_step);
        destination_trace_data->level_min_index = level_index;
        destination_trace_data->level_max_index = level_index;
        destination_trace_data->level_min = (double)*source_iterator;
        destination_trace_data->level_max = (double)*source_iterator;

        while (source_iterator < source_end_pointer) {
          // copy the level data
            *target_iterator = *source_iterator;

          // compute the level index
            level_index = (int)round((level_max-(double)*target_iterator)/destination_channel_data->level_step);
          // Increment value at the power signature array at the computed position
            destination_channel_data->power_signature[frequency_index+(level_index*destination_trace_data->number_frequency_bins)]++;
          // add value to the normal distributioin computation for level
            destination_channel_data->level_normal[frequency_index].add_element((double)*target_iterator);
          // update maximum and minimul level references for the channel 
            if ( *target_iterator < destination_trace_data->level_min ) {
              destination_trace_data->level_min = *target_iterator;
              destination_trace_data->level_min_index = level_index;
            }
            if ( *target_iterator > destination_trace_data->level_max ) {
              destination_trace_data->level_max = *target_iterator;
              destination_trace_data->level_max_index = level_index;
            }

          // Erase the source trace infomation 
            *source_iterator = NULL_SPECTRUM;

          // Increment iterators, skipping cases where, even after increment, source stilll pointing to a NULL_SPECTRUM, meaning that it reached an interception with adjascent channel
            while ( *source_iterator == NULL_SPECTRUM) {
              frequency_index++;
              source_iterator++;
              target_iterator++;
              if (source_iterator == source_end_pointer) break;
            }
        }

      // Increment the number of time bins on the destination to include the new trace
        destination_trace_data->number_time_bins++;

      // increment the counter for the number of channel samples
        destination_channel_data->number_of_samples++;
        
      // set the trace data with the same noise reference. Necessary in case that multiple streams are available on the file
        destination_trace_data->noise_reference = noise_reference;
    }

  // Method to copy segment of a trace into another and delete the original trace data, updating channel information in the process.
    void abstract_noise(
            // Reference stream thread ID
              double reference_thread_index,
            // Channel data object that will receive the moved data at the end postion
              channel_data *destination_channel_data) {

    // Update basic information on channel data for the noise
      destination_channel_data->stream_code = reference_thread_index;
      destination_channel_data->number_of_samples = number_time_bins;

      destination_channel_data->begin_status = channel_status();
      destination_channel_data->begin_status.status = false;
      destination_channel_data->begin_status.posix_time = posix_time[0];
      destination_channel_data->begin_status.nanosecond_time = nanosecond_time[0];
      destination_channel_data->begin_status.duration = ((double)posix_time[number_time_bins]-(double)posix_time[0]) + (((double)nanosecond_time[number_time_bins]-(double)nanosecond_time[0])/NANOSECONDS_IN_SECOND);

      destination_channel_data->end_status = channel_status();
      destination_channel_data->end_status.status = false;
      destination_channel_data->end_status.posix_time = posix_time[number_time_bins];
      destination_channel_data->end_status.nanosecond_time = nanosecond_time[number_time_bins];
      destination_channel_data->end_status.duration = 0;

    // reset index variables
      level_min_index = 0;
      level_max_index = UINT32_MAX;

    // Update spectrogram infomation values taking into consideration the discarted channels, computing minimum and maximum values during this process
      float *spectrogram_iterator = spectrogram;
      float *end_pointer = &spectrogram[number_time_bins*number_frequency_bins];

    // Copy the original channel maximum as reference before updating to the actual maximum value
      double reference_level = level_max;
      level_max = (double)*spectrogram_iterator;
      level_min = (double)*spectrogram_iterator;

    // compute a reference index based on the first level and reset maximum and minimum values
      uint32_t level_index = (int)round((reference_level-(double)*spectrogram_iterator)/destination_channel_data->level_step);
      level_min_index = level_index;
      level_max_index = level_index;

    // reset frequency index to point at the first element on the vector
      uint32_t frequency_index = 0;

    // compute spectrum profile and level range for the noise data
      while (spectrogram_iterator < end_pointer) {
        // if the value at the current iterator is not null
          if ( *spectrogram_iterator != NULL_SPECTRUM) {
            // compute the level index
              level_index = (int)round((reference_level-(double)*spectrogram_iterator)/destination_channel_data->level_step);

            // Increment value at the power signature array at the computed position
              destination_channel_data->power_signature[frequency_index+(level_index*number_frequency_bins)]++;

// TODO: Replace the NULL_SPECTRUM by the noise reference for the trace     

            // update maximum and minimum level references for the channel. Scale is reversed, the minimum level is has the maximum index
              if ( *spectrogram_iterator > level_max ) {
                level_max = *spectrogram_iterator;
                level_max_index = level_index;
              }
              if ( *spectrogram_iterator < level_min ) {
                level_min = *spectrogram_iterator;
                level_min_index = level_index;
              }
          }

        // Increment iterators
          frequency_index++;
          spectrogram_iterator++;

        // reset the frequency index when it reaches the maximum
          if (frequency_index == number_frequency_bins) frequency_index = 0;
      }
    }

    void shrink_to_fit(
            // 
              uint32_t minimum_level_index,
            // trace that contains the segment to be moved
              uint32_t maximum_level_index) {
      // TODO: Create a method to reduce memory usage by reducing the arrays to fit the data exactly

              }

};

// Global variables used to store recovered data
  position site;
  equipment_data equipment;
  unordered_map<double,string> thread_index;
  unordered_map<double,trace_data> stream;
  unordered_map<double,sweep_parameters> stream_sweep; // Global variable used to store detection parmeters to all frequency bins within a stream
  vector<event_item> event_log;
  vector<channel_data> channel_list;
  non_normal time_offset; // offset in seconds between GPS time and Machine time

// Detect if machine uses big endian or little endian. Bin files are little endian
bool Is_Big_Endian(void) {
    union {
        uint32_t i;
        char c[4];
    } big_endian_int = {0x01020304};

    return (big_endian_int.c[0] == 1); 
}

// Finishes processing handling possible error and printing log as json output
// TODO: This function should be modified to print in a log file or the output should be redirected to do so by the caller function. If so. this should be modified to include additional references to the events such as file, date and time
void Finish(bool all_ok, const std::string& output_message) {
    int i = 0;
    bool first = true;

  // free memory allocated
    delete[] memblock;

  // 
    if(all_ok){
      // TODO: move file to done_box
      // Print output messages for logging purposes
        // sweep through all block types and number of blocks processed for each type
        for (i=0;i<NUMBER_OF_BLOCK_TYPES;i++) {
          if(block_type_count[BLOCK_CODE[i]] > 0) {
            if(first) {
              output_log << "\n\t\t\"Success\": {\n\t\t\t\""; 
              first = false;
            }
            else {
              output_log << ",\n\t\t\t\"";
            }
            output_log << dec;
            output_log << BLOCK_NAME[i] << "\": " << block_type_count[BLOCK_CODE[i]];
          }
        }     
        output_log << "\n\t\t}\n\t}";

    } else{
      // TODO: move file to error_box - Not necessary to test if processing is finished
      // Print output messages for logging purposes
        output_log << "\n\t\t\"Error\": \"" << output_message << "\"\n\t}";
    }
}

// Test block end and move cursor
char * Test_Block(char * block_start, int32_t *block_header, string block_type) {
  // define local variables
    uint8_t * mem_cursor;
    int32_t bytes_in_block;
    int32_t block_checksum;
    int i;
    int32_t checksum = 0;

  // get the number of data bytes in the block 
    bytes_in_block = block_header[1];

  //compute checksum adding all values in bytes from the first (including header) to the last excluding trailer. (move the mem_cursor along)
    for(mem_cursor=reinterpret_cast<uint8_t *>(block_start);mem_cursor<reinterpret_cast<uint8_t *>(block_start)+bytes_in_block+BLOCK_HEADER_SIZE;mem_cursor++) {
      checksum = checksum + *mem_cursor; 
    } 

  // read the checksum information recorded on the first four bytes of block trailer 
    block_checksum = *(reinterpret_cast<int32_t *>(mem_cursor));

  // confirm checksum
    if (checksum != block_checksum) {
      output_log << "\n\t\t\"Warning\": \"block with type " << block_type << " with checksum not matching. Position: 0x" << hex << block_start-memblock << "\",";
    }
    
  // test if the end of the block (last 32 bits on the block) was reached by verifying the "end of block" marker ( 0x55555555 )
    mem_cursor = mem_cursor+INT32T_NUMBER_SIZE;
    if ( *(reinterpret_cast<int32_t *>(mem_cursor)) != BLOCK_END_MARKER) {
      output_log << "\n\t\t\"Warning\": \"Not able to detect the end of a block with type " << block_type << " at position: 0x" << hex << block_start-memblock << "\",";
    }
        
  // move to pointer to the next block
    mem_cursor = mem_cursor+INT32T_NUMBER_SIZE;
    return (reinterpret_cast<char *>(mem_cursor));
}

// Receive pointer to transform original CRFS BIN clock format ( dd mm yy null hh mm ss cc ) into timespec structures. Return pointer to the byte after the transformed data 
char * Get_Clock(char * mem_cursor, struct timespec * clock_datetime, bool is_nanosecond) {
  // define local variables
    unsigned char *number_uchar;
    uint32_t *number_uint32;
    struct tm datetime = {0};

  // get pointer as unsigned char and move the original pointer to the next data field
    number_uchar = reinterpret_cast<unsigned char *>(mem_cursor);
    mem_cursor = mem_cursor + DATE_SIZE;
  
  // extract the date and time information from some of the first 8 bytes and mount it in a tm structure
    datetime.tm_mday = (int)number_uchar[0];
    datetime.tm_mon = (int)number_uchar[1] - 1;
    datetime.tm_year = (int)number_uchar[2] + 100;
    datetime.tm_hour = (int)number_uchar[4];
    datetime.tm_min = (int)number_uchar[5];
    datetime.tm_sec = (int)number_uchar[6];

  // convert the tm structure to Unix time into a timespec structure
    (*clock_datetime).tv_sec = mktime(&datetime);

  // if the data contains nanosecond information
    if (is_nanosecond){
      // if the last byte on the first 8 contains hundredths of a second information, there is no additional nanosecond information and this should be considered
        if(number_uchar[7] > 0) {
          (*clock_datetime).tv_nsec = number_uchar[7]*10e-7;
        } else {
      // else, take the following 4 bytes as nanosecond information and move accordingly the memory cursor pointer
      // TODO: Detected difference in the nanosecond range. For the sample file, CRFS reader indicates 351619005 ns and this program reads 350555000
          number_uint32 = reinterpret_cast<uint32_t *>(mem_cursor);
          (*clock_datetime).tv_nsec = *number_uint32;
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
        }
    } else {
      (*clock_datetime).tv_nsec = 0;
    }
  
  return mem_cursor;
} 

// Processing First Bytes of Spectrum Block type 60 or 63
char * Decode_Spectrum_Block_Header(
    // pointer to the input data position on memory
      char * mem_cursor,
    // thread identifier as specified on the bin file
      double thread_identifier,
    // number of bytes of the file
      uint32_t data_size,
    // data offset defined
      double *decode_data_offset) {

  // SPECTRUN DATA
    struct timespec spectrum_clock;
    double start_frequency;
    double stop_frequency;
    uint16_t start_channel;
    uint16_t stop_channel; 
    uint32_t sample_duration; // time in nanoseconds for direct compatibility with time.h 
    uint32_t number_of_loops;
    string antenna; // unit info + datetime + antenna number'
// TODO: Check if antenna can change in the middle of the scan if used a manual configuratiom with automatic antenna selection. The automatic system should use fixed gain and fixed antenna
    uint8_t processing;
    uint8_t measurement_unit;
    int8_t data_offset;
    char global_error_code;
    char global_clipping_flag;
    uint8_t spectrum_group_id;
    uint16_t number_tuning;
    uint16_t number_agc;
    uint32_t number_spectrum_data;

    struct tuning_block {
      uint16_t frequency;
      uint8_t clipping_flags;
      uint8_t radio_error;
    };

    tuning_block *tuning_block_file_iterator;
    uint8_t *agc_file_iterator;

  // get the date time information from de wall clock fields (12 bytes with date and time down to nanosecond) and move mem_cursor to the next data field
    mem_cursor = Get_Clock(mem_cursor, &spectrum_clock, true);

  // test if this is the first time that the current stream includes spectral data
    if (stream.find(thread_identifier) == stream.end()) {

      // If it is the first time, get the essential data and initialize the stream
      // store initial memory position for block size estimation
        char * initial_mem_cursor = mem_cursor; 

      // get the start frequency information in hertz and move mem_cursor to the next data field
        start_frequency = ((double)*(reinterpret_cast<uint16_t *>(mem_cursor))*(double)1e6)+((double)*(reinterpret_cast<uint32_t *>(mem_cursor+2))*(double)1e3); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE + INT32T_NUMBER_SIZE;

      // get the stop frequency information in hertz and move mem_cursor to the next data field
        stop_frequency = ((double)*(reinterpret_cast<uint16_t *>(mem_cursor))*(double)1e6)+((double)*(reinterpret_cast<uint32_t *>(mem_cursor+2))*(double)1e3); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE + INT32T_NUMBER_SIZE;

      // get the start channel information and move mem_cursor to the next data field
        start_channel = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // get the stop channel information and move mem_cursor to the next data field
        stop_channel = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // get the sample duration information in microsecond, store it in nanosecond and move mem_cursor to the next data field
        sample_duration = *(reinterpret_cast<uint32_t *>(mem_cursor))*1000; 
        mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
      
      // get the number of measurement loops information and move mem_cursor to the next data field
        number_of_loops = *(reinterpret_cast<uint32_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

      // get the antenna number information, merge it with host and datetime information and move mem_cursor to the next data field
        antenna = to_string(*(reinterpret_cast<uint8_t *>(mem_cursor))) + string("@") + equipment.hostname + string("@") + to_string(spectrum_clock.tv_sec); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;
      
      // get the trace processing information (0 = single measurement, 1 = average, 2 = peak, 3 = minimum) and move mem_cursor to the next data field
        processing = *(reinterpret_cast<uint8_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;        

      // get the measurement unit information (1 = dBm, 2 = dBμV/m) and move mem_cursor to the next data field
        measurement_unit = *(reinterpret_cast<uint8_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;        
      
      // get the data offset information and move mem_cursor to the next data field
        data_offset = *(reinterpret_cast<int8_t *>(mem_cursor)); 
        *decode_data_offset = (double)data_offset;
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the global error code information and move mem_cursor to the next data field
        global_error_code = *(mem_cursor); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the global clipping flag  information and move mem_cursor to the next data field
        global_clipping_flag = *(mem_cursor);
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the spectrum group id information and move mem_cursor to the next data field
        spectrum_group_id = *(reinterpret_cast<uint8_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the number of tunning blocks and move mem_cursor to the next data field
        number_tuning = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // get the number of agc values and move mem_cursor to the next data field
        number_agc = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // skip the number of padding bytes information and move mem_cursor to the next data field
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;  

      // get the number spectrun data points and move mem_cursor to the next data field
        number_spectrum_data = *(reinterpret_cast<uint32_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

      // estimate the total number of rows based on the dataset size and current block information (overestimate to files with multiple stream)
        int block_memory_size = (initial_mem_cursor - mem_cursor) + ( number_tuning * TUNING_BLOCK_SIZE ) + number_agc + number_spectrum_data;

      // initialize detection parameter structure for the current stream in order to buid the antenna factor array
        stream_sweep[thread_identifier] = sweep_parameters(
          start_frequency,
          stop_frequency,
          number_spectrum_data,
          measurement_unit,
          (int)output_unit,
          antenna);

      // initialize trace_data structure for the current stream and estimating the number of timebins according to the file size and current block size
        stream[thread_identifier] = trace_data(
          number_of_loops,
          sample_duration,
          number_spectrum_data,
          int(round(double(data_size) / double(block_memory_size+12))),
          start_frequency,
          stop_frequency,
          spectrum_clock,
          false);
    }
    else {
      // If not the first time that this stream receives spectrum data

      // store new time on the stream
        stream[thread_identifier].posix_time[stream[thread_identifier].number_time_bins] = (uint32_t)spectrum_clock.tv_sec;
        stream[thread_identifier].nanosecond_time[stream[thread_identifier].number_time_bins] = (uint32_t)spectrum_clock.tv_nsec;
        
      // Skip data that dont change between blocks of the same stream in the same file
        mem_cursor = mem_cursor + (INT8T_NUMBER_SIZE*3)+(INT32T_NUMBER_SIZE*4)+(INT16T_NUMBER_SIZE*4);

      // get the data offset information and move mem_cursor to the next data field
        data_offset = *(reinterpret_cast<int8_t *>(mem_cursor));
        *decode_data_offset = (double)data_offset;
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // Get data that might have changed or is needed for the processing
      // get the global error code information and move mem_cursor to the next data field
        global_error_code = *(mem_cursor); 
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the global clipping flag  information and move mem_cursor to the next data field
        global_clipping_flag = *(mem_cursor);
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // skip the spectrum group id information and move mem_cursor to the next data field
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

      // get the number of tunning blocks and move mem_cursor to the next data field
        number_tuning = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // get the number of agc values and move mem_cursor to the next data field
        number_agc = *(reinterpret_cast<uint16_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

      // skip the number of padding bytes information and move mem_cursor to the next data field
        mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;  

      // get the number spectrun data points and move mem_cursor to the next data field
        number_spectrum_data = *(reinterpret_cast<uint32_t *>(mem_cursor)); 
        mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
    }

  // get the the pointer to the tunning block values and move mem_cursor to the next data field
    tuning_block_file_iterator = reinterpret_cast<tuning_block *>(mem_cursor);
    mem_cursor = mem_cursor + ( number_tuning * TUNING_BLOCK_SIZE );

  // get the the pointer to the agc values and move mem_cursor to the next data field
    agc_file_iterator = reinterpret_cast<uint8_t *>(mem_cursor);
    mem_cursor = mem_cursor + number_agc;

  // return the memory position of the input data at the end of the processing
    return mem_cursor;
}

#define update_level_range() \
    double maximum_expected_value = DEFAULT_LEVEL_STEP*ceil(((BYTE_MAX_VALUE/STEP_PER_LEVEL_UNIT) + *data_offset + stream_sweep[thread_identifier].level_offset_factor_max)/DEFAULT_LEVEL_STEP); \
    double minimum_expected_value = DEFAULT_LEVEL_STEP*floor(((0/STEP_PER_LEVEL_UNIT) + *data_offset + stream_sweep[thread_identifier].level_offset_factor_min)/DEFAULT_LEVEL_STEP); \
    if ( stream[thread_identifier].number_time_bins == 0 ) { \
      stream[thread_identifier].level_max = maximum_expected_value; \
      stream[thread_identifier].level_min = minimum_expected_value; \
    } \
    else { \
      if ( stream[thread_identifier].level_max < maximum_expected_value ) stream[thread_identifier].level_max = maximum_expected_value; \
      if ( stream[thread_identifier].level_min < minimum_expected_value ) stream[thread_identifier].level_min = minimum_expected_value; \
    };


// Takes the pointer to the beginning of spectrum data in blocks type 60 or 63
char * Get_Spectrum_No_Detection(char * mem_cursor, double thread_identifier, uint32_t data_size) {

    double *data_offset = new double;
    mem_cursor = Decode_Spectrum_Block_Header(mem_cursor, thread_identifier, data_size, data_offset);

  // update the range values for the level taking into consideration the data_offset
    update_level_range();

  // get the the pointer to spectrum data values from the original bin file
    uint8_t *spectrum_input_iterator;
    spectrum_input_iterator = reinterpret_cast<uint8_t *>(mem_cursor);

  // set the output pointer to store data over the main stream storage
    float *spectrum_output_iterator;
    spectrum_output_iterator = &stream[thread_identifier].spectrogram[(stream[thread_identifier].number_frequency_bins*stream[thread_identifier].number_time_bins)];

  // set pointer to the offset factor iterator
    double * level_offset_factor_iterator = stream_sweep[thread_identifier].level_offset_factor;

  // set pointer control to the end of the spectrum output vector;
    float * pointer_end = &spectrum_output_iterator[stream[thread_identifier].number_frequency_bins];

  // sweep the input and antenna factor vectors computing the output vector resulting values  
    do {
        *spectrum_output_iterator = (float)(*spectrum_input_iterator)/STEP_PER_LEVEL_UNIT + *data_offset + (float)*level_offset_factor_iterator;
        spectrum_input_iterator++;
        level_offset_factor_iterator++;
        spectrum_output_iterator++;
    } while ( spectrum_output_iterator < pointer_end );

  // register the new trace
    stream[thread_identifier].number_time_bins++;

  // update memory cursor to the end of spectrum data field
    mem_cursor = mem_cursor + stream[thread_identifier].number_frequency_bins;

    return mem_cursor;
} 

// Takes the pointer to the beginning of spectrum data in blocks type 60 or 63
char * Get_Spectrum_Level_Detection(char * mem_cursor, double thread_identifier, int data_size) {

    double *data_offset = new double;
    mem_cursor = Decode_Spectrum_Block_Header(mem_cursor, thread_identifier, data_size, data_offset);
  
  // update the range values for the level taking into consideration the data_offset
    update_level_range();

  // get the the pointer to spectrum data values from the original bin file
    uint8_t *spectrum_input_iterator = reinterpret_cast<uint8_t *>(mem_cursor);

  // construct the object to perform the detection
    signal signal_data(spectrum_input_iterator,
                       stream_sweep[thread_identifier].level_offset_factor,
                       stream_sweep[thread_identifier].number_frequency_bins,
                       STEP_PER_LEVEL_UNIT,
                       *data_offset,
                       &stream[thread_identifier].spectrogram[(stream[thread_identifier].number_frequency_bins*stream[thread_identifier].number_time_bins)],
                       &stream[thread_identifier].spectrogram_sum[(stream[thread_identifier].number_frequency_bins*stream[thread_identifier].number_time_bins)]);
  
  // perform the signal correction and detection.
    signal_data.run_detector(stream_sweep[thread_identifier].detection_threshold,
                             stream_sweep[thread_identifier].window_size,
                             stream_sweep[thread_identifier].second_window_offset,
                             stream_sweep[thread_identifier].minimum_ascending_break_separation,
                             stream_sweep[thread_identifier].minimum_descending_break_separation,
                             stream_sweep[thread_identifier].minimum_channel_width,
                             &stream[thread_identifier].threshold_crossing,
                             &stream[thread_identifier].noise_reference,
                             (long)stream[thread_identifier].number_time_bins);

/* // ! Plot for debugging 
    if ( 773*(int)round((double)stream[thread_identifier].number_time_bins/773) == stream[thread_identifier].number_time_bins ) {

      // sort the threshold_crossing vector
        sort( stream[thread_identifier].threshold_crossing.begin( ), stream[thread_identifier].threshold_crossing.end( ));


        // signal_data.plot_detection(0,stream[thread_identifier].number_frequency_bins,25,stream[thread_identifier].number_time_bins,&stream[thread_identifier].threshold_crossing);
        // signal_data.plot_detection(40090-125, 40090+125, 1, stream[thread_identifier].number_time_bins, &stream[thread_identifier].threshold_crossing);
        // signal_data.plot_detection(18150,21400,1,stream[thread_identifier].number_time_bins,&stream[thread_identifier].threshold_crossing);
    }
*/

  // register the new trace
    stream[thread_identifier].number_time_bins++;

  // update memory cursor to the end of spectrum data field
    mem_cursor = mem_cursor + stream[thread_identifier].number_frequency_bins;

    return mem_cursor;
} 

// Read a data block. Takes the pointer to the block first byte and return the pointer to the next block
char * Block_Read(char * mem_cursor) {
  // define local variables
    int32_t *block_header;
    int32_t free_text_length;
    char * block_start = mem_cursor;
    int i;

    string str1;
    string str2;

  // define variables used to store recovered data
    // ALL
      double thread_identifier;

    // V3_UNIT_AND_JOB_INFORMATION
      // see global variables

    // V3_DATA_THREAD_INFORMATION
      string v3_data_thread_info;

    // V3_FREE_TEXT_INFORMATION
      string free_text_id;
      string free_text;

    // V4_DATA_THREAD_INFORMATION
      int32_t data_thread_group_id;
      string data_thread_text;

    // TIMED_FREE_TEXT_INFORMATION
      string timed_txt_identifier;
      string timed_text;

    // V3_GPS_DATA
      struct timespec wall_clock; // wall clock
      struct timespec satellite_clock; // clock returned from the gps module
      char fix_status;
      char satellite_view;
      float heading;

  // Create 32bit int pointer to the data to decode the header ( 3 int numbers with 32bits each)
    block_header = reinterpret_cast<int32_t *>(mem_cursor);

  // Get thread identifier information from the first number in the block header
    thread_identifier = (double)block_header[0];
  // TODO: Handle error possible due to the thread_identifier being greater than the maximum acceptable value = MAXIMUM_ACCEPTABLE_THREAD_ID

  // save reference of the first byte of the block and move cursor to the first data field
    block_start = mem_cursor;
    mem_cursor = mem_cursor + BLOCK_HEADER_SIZE; 

  // act according to the block type, described in the third and last number in the block header
    switch (block_header[2]) {
      case FILE_FOOTER :
        // update block type processing log
          block_type_count[FILE_FOOTER] = block_type_count[FILE_FOOTER] + 1;

        // ! PROCESSING TO BE DEVELOPED.

        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for FILE_FOOTER type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "FILE_FOOTER");

		    break;
	    case V2_UNIT_INFORMATION :
        // update block type processing log
          block_type_count[V2_UNIT_INFORMATION] = block_type_count[V2_UNIT_INFORMATION] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_UNIT_INFORMATION type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_UNIT_INFORMATION");  

        break;
      case V3_UNIT_AND_JOB_INFORMATION :
        // update block type processing log
          block_type_count[V3_UNIT_AND_JOB_INFORMATION] = block_type_count[V3_UNIT_AND_JOB_INFORMATION] + 1;

        // get the unit hostname information and move cursor to the next data field. Terminate reading avoiding the 0 pading values
          // get the minimum amount of text that is known to be in this field
            equipment.hostname.assign(mem_cursor,MINIMUM_HOSTNAME_SIZE);
            i = MINIMUM_HOSTNAME_SIZE;

          // get the following caracters one by one until a padding NULL char is found or the maximum number of bytes is found.
            while ( (mem_cursor[i] != PADDING) && (i < HOSTNAME_SIZE) ) {
              equipment.hostname += mem_cursor[i];
              i++;
            }
          
          mem_cursor = mem_cursor + HOSTNAME_SIZE;

        // get the unit info information and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          equipment.unit_info.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;
        
        // get the method information and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          equipment.method.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_UNIT_AND_JOB_INFORMATION");

		    break;
      case V2_DATA_THREAD_INFORMATION :
        // update block type processing log
          block_type_count[V2_DATA_THREAD_INFORMATION] = block_type_count[V2_DATA_THREAD_INFORMATION] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_DATA_THREAD_INFORMATION type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_DATA_THREAD_INFORMATION");  

        break;
	    case V3_DATA_THREAD_INFORMATION :
        // update block type processing log
          block_type_count[V3_DATA_THREAD_INFORMATION] = block_type_count[V3_DATA_THREAD_INFORMATION] + 1;

        // get the v3_data_thread_info and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          v3_data_thread_info.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;

        // store data thread information on the thread index
          thread_index[thread_identifier]=v3_data_thread_info;

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_DATA_THREAD_INFORMATION");          
		    
        break;
	    case V2_FREE_TEXT_INFORMATION :
        // update block type processing log
          block_type_count[V2_FREE_TEXT_INFORMATION] = block_type_count[V2_FREE_TEXT_INFORMATION] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_FREE_TEXT_INFORMATION type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_FREE_TEXT_INFORMATION");  

        break;
      case V3_FREE_TEXT_INFORMATION :
        // update block type processing log
          block_type_count[V3_FREE_TEXT_INFORMATION] = block_type_count[V3_FREE_TEXT_INFORMATION] + 1;

        // get the free text id information and move cursor to the next data field
          // get the minimum amount of text that is known to be in this field
            free_text_id.assign(mem_cursor, MINIMUM_FREE_TEXT_ID_SIZE);
            i = MINIMUM_FREE_TEXT_ID_SIZE;

          // get the following caracters one by one until a padding NULL char is found or the maximum number of bytes is found.
            while ( (mem_cursor[i] != PADDING) && (i < FREE_TEXT_ID_SIZE) ) {
              equipment.hostname += mem_cursor[i];
              i++;
            }

          mem_cursor = mem_cursor + FREE_TEXT_ID_SIZE;
        
        // get the unit info information and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          free_text.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_FREE_TEXT_INFORMATION");          
		    
		    break;
	    case V4_DATA_THREAD_INFORMATION :
        // update block type processing log
          block_type_count[V4_DATA_THREAD_INFORMATION] = block_type_count[V4_DATA_THREAD_INFORMATION] + 1;

        // get the group id information and move mem_cursor to the next data field
          data_thread_group_id = *(reinterpret_cast<int32_t *>(mem_cursor)); 
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

        // get the method information and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          data_thread_text.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;

        // store data thread information on the thread index
          thread_index[thread_identifier]=data_thread_text;

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V4_DATA_THREAD_INFORMATION");          
		    
		    break;
	    case V2_GPS_DATA :
        // update block type processing log
          block_type_count[V2_GPS_DATA] = block_type_count[V2_GPS_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_GPS_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_GPS_DATA");  

        break;
      case V3_GPS_DATA :
        // update block type processing log
          block_type_count[V3_GPS_DATA] = block_type_count[V3_GPS_DATA] + 1;

        // get the date time information from de wall clock fields (12 bytes with date and time down to nanosecond) and move mem_cursor to the next data field
          mem_cursor = Get_Clock(mem_cursor, &wall_clock, true);          
          
        // get the date time information from de gps clock fields (8 bytes with date and time down to second) and move mem_cursor to the next data field
          mem_cursor = Get_Clock(mem_cursor, &satellite_clock, false);
          
        // get the  fix_status (1 byte)  If status=1: 0=No Fix, 1=Standard GPS, 2=Differential GPS If status=0: set to zero and move to the next data field
          fix_status = *(reinterpret_cast<uint8_t *>(mem_cursor)); 
          mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

        // get the satellite_view (1 byte) 0=bad, 1+ better and move to the next data field
          satellite_view = *(reinterpret_cast<uint8_t *>(mem_cursor)); 
          mem_cursor = mem_cursor + INT8T_NUMBER_SIZE;

        // continue decoding if the GPS information is reliable
          if ( (fix_status > 0) && (satellite_view>3) ){

            // test if both gps and satellite clocks are the same. If not apply correction in order to keep all time as UTC
              double current_time_offset = difftime(satellite_clock.tv_sec,wall_clock.tv_sec);
              if ( current_time_offset > double(0.0)) {
                // output message 
                  output_log << "\n\t\t\"Warning\": \"Meter system clock and GPS Clock not matching: (System-GPS) = " << current_time_offset << "\",";
                  time_offset.add_element(float(current_time_offset));
              }

            // store date
              site.add_time(wall_clock);

            // get the  heading (2 bytes) in degrees * 100 and move to the next data field
              heading = (float)(*(reinterpret_cast<uint16_t *>(mem_cursor)))/100.0; 
              mem_cursor = mem_cursor + INT16T_NUMBER_SIZE;

            // get the  latitude (4 bytes) in degrees * 1000000: +ve=N, -ve=S and move to the next data field
              site.latitude.add_element((double)(*(reinterpret_cast<int32_t *>(mem_cursor)))/1000000.0); 
              mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

            // get the  longitude (4 bytes) in degrees * 1000000: +ve=E, -ve=W and move to the next data field
              site.longitude.add_element((double)(*(reinterpret_cast<int32_t *>(mem_cursor)))/1000000.0); 
              mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

            // get the  speed (4 bytes) in kph * 1000 and move to the next data field
              site.horizontal_variation.add_element((double)(*(reinterpret_cast<int32_t *>(mem_cursor)))/1000.0); 
              mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

            // get the  altitude (4 bytes) in Metres * 1000 and move to the next data field
              site.altitude.add_element((double)(*(reinterpret_cast<int32_t *>(mem_cursor)))/1000.0); 
              mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;

            // test the checksum of the block and if it was properly closed, then move cursor to the next block
              mem_cursor = Test_Block(block_start, block_header, "V3_GPS_DATA");          
          }		    
		    break;
	    case V3_TIMED_FREE_TEXT_INFORMATION :
        // update block type processing log
          block_type_count[V3_TIMED_FREE_TEXT_INFORMATION] = block_type_count[V3_TIMED_FREE_TEXT_INFORMATION] + 1;

        // get the date time information from de wall clock fields (12 bytes with date and time down to nanosecond) and move mem_cursor to the next data field
          mem_cursor = Get_Clock(mem_cursor, &wall_clock, true);
                    
        // get the Data Type Identifier information and move mem_cursor to the next data field. Terminate reading avoiding the 0 pading values
          // get the minimum amount of text that is known to be in this field
            timed_txt_identifier.assign(mem_cursor,MINIMUM_TIMED_TEXT_SIZE);
            i = MINIMUM_TIMED_TEXT_SIZE;

          // get the following caracters one by one until a padding NULL char is found or the maximum number of bytes is found.
            while ( (mem_cursor[i] != PADDING) && (i < TIMED_TEXT_SIZE) ) {
              timed_txt_identifier += mem_cursor[i];
              i++;
            }
          
          mem_cursor = mem_cursor + TIMED_TEXT_SIZE;

        // get the text information and move cursor to the next data field
          free_text_length = *(reinterpret_cast<int32_t *>(mem_cursor));
          mem_cursor = mem_cursor + INT32T_NUMBER_SIZE;
          timed_text.assign(mem_cursor, free_text_length);
          mem_cursor = mem_cursor + free_text_length;

        // store data thread information on the thread index
          event_log.push_back(event_item(wall_clock, timed_txt_identifier, timed_text));

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_TIMED_FREE_TEXT_INFORMATION");          
		    
		    break;
	    case V5_TIMED_FREE_TEXT_INFORMATION :
        // update block type processing log
          block_type_count[V5_TIMED_FREE_TEXT_INFORMATION] = block_type_count[V5_TIMED_FREE_TEXT_INFORMATION] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_TIMED_FREE_TEXT_INFORMATION type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_TIMED_FREE_TEXT_INFORMATION");  

        break;
      case V2_8_BIT_SPECTRAL_DATA :
        // update block type processing log
          block_type_count[V2_8_BIT_SPECTRAL_DATA] = block_type_count[V2_8_BIT_SPECTRAL_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_8_BIT_SPECTRAL_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_8_BIT_SPECTRAL_DATA");  

        break;
      case V3_8_BIT_SPECTRAL_DATA :
         // update block type processing log
          block_type_count[V3_8_BIT_SPECTRAL_DATA] = block_type_count[V3_8_BIT_SPECTRAL_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V3_8_BIT_SPECTRAL_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_8_BIT_SPECTRAL_DATA");          
		    
		    break;
	    case V2_THRESHOLD_COMPRESSED_DATA :
        // update block type processing log
          block_type_count[V2_THRESHOLD_COMPRESSED_DATA] = block_type_count[V2_THRESHOLD_COMPRESSED_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_THRESHOLD_COMPRESSED_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_THRESHOLD_COMPRESSED_DATA");  

        break;
      case V3_THRESHOLD_COMPRESSED_DATA :
         // update block type processing log
          block_type_count[V3_THRESHOLD_COMPRESSED_DATA] = block_type_count[V3_THRESHOLD_COMPRESSED_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V3_THRESHOLD_COMPRESSED_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_THRESHOLD_COMPRESSED_DATA");          
		    
		    break;
	    case V4_THRESHOLD_COMPRESSED_DATA :
        // update block type processing log
          block_type_count[V4_THRESHOLD_COMPRESSED_DATA] = block_type_count[V4_THRESHOLD_COMPRESSED_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.
        
        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V4_THRESHOLD_COMPRESSED_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V4_THRESHOLD_COMPRESSED_DATA");          
		    
		    break;
      case V5_THRESHOLD_COMPRESSED_DATA :
        // update block type processing log
          block_type_count[V5_THRESHOLD_COMPRESSED_DATA] = block_type_count[V5_THRESHOLD_COMPRESSED_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_THRESHOLD_COMPRESSED_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_THRESHOLD_COMPRESSED_DATA");  

        break;
      case V2_FREQUENCY_CHANNEL_OCCUPANCY_SM1793 :
        // update block type processing log
          block_type_count[V2_FREQUENCY_CHANNEL_OCCUPANCY_SM1793] = block_type_count[V2_FREQUENCY_CHANNEL_OCCUPANCY_SM1793] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_FREQUENCY_CHANNEL_OCCUPANCY_SM1793 type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_FREQUENCY_CHANNEL_OCCUPANCY_SM1793");  

        break;
      case V3_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 :
        // update block type processing log
          block_type_count[V3_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] = block_type_count[V3_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] + 1;

        //  ! PROCESSING TO BE DEVELOPED.
        
        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V3_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it was properly closed, then move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V3_FREQUENCY_CHANNEL_OCCUPANCY_SM1880");          
		    
		    break;
	    case V4_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 :
        // update block type processing log
          block_type_count[V4_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] = block_type_count[V4_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] + 1;

        //  ! PROCESSING TO BE DEVELOPED.
        
        // output message 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V4_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V4_FREQUENCY_CHANNEL_OCCUPANCY_SM1880");          
		    
		    break;
      case V5_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 :
        // update block type processing log
          block_type_count[V5_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] = block_type_count[V5_FREQUENCY_CHANNEL_OCCUPANCY_SM1880] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_FREQUENCY_CHANNEL_OCCUPANCY_SM1880 type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_FREQUENCY_CHANNEL_OCCUPANCY_SM1880");  

        break;
      case V4_SPECTRAL_DATA :
        // TODO FINISH THIS 
        
        // update block type processing log
          block_type_count[V4_SPECTRAL_DATA] = block_type_count[V4_SPECTRAL_DATA] + 1;      

        // get the spectrum data
          switch (detection_mode) {
          // no detection simple store the data for later processing
            case NO_DETECTION_OPTION : {
              mem_cursor = Get_Spectrum_No_Detection(mem_cursor,thread_identifier,(uint32_t)data_size);
              break;
            }
            case LEVEL_DETECTION_OPTION : {
              mem_cursor = Get_Spectrum_Level_Detection(mem_cursor,thread_identifier,data_size);
              break;
            }
          }
        
        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V4_SPECTRAL_DATA");          
		    
		    break;
	    case V5_SPECTRAL_DATA :
        // update block type processing log
          block_type_count[V5_SPECTRAL_DATA] = block_type_count[V5_SPECTRAL_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_SPECTRAL_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_SPECTRAL_DATA");  

        break;
      case V5_CLASSIFIER_DATA :
        // update block type processing log
          block_type_count[V5_CLASSIFIER_DATA] = block_type_count[V5_CLASSIFIER_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_CLASSIFIER_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_CLASSIFIER_DATA");  

        break;
      case V2_16_BIT_IQ_DATA :
        // update block type processing log
          block_type_count[V2_16_BIT_IQ_DATA] = block_type_count[V2_16_BIT_IQ_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V2_16_BIT_IQ_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V2_16_BIT_IQ_DATA");  

        break;
      case V5_16_BIT_IQ_DATA :
        // update block type processing log
          block_type_count[V5_16_BIT_IQ_DATA] = block_type_count[V5_16_BIT_IQ_DATA] + 1;

        //  ! PROCESSING TO BE DEVELOPED.

        // output warning message of module not developed 
          output_log << "\n\t\t\"Warning\": \"Decoding not available for V5_16_BIT_IQ_DATA type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

        // test the checksum of the block and if it is ok, move cursor to the next block
          mem_cursor = Test_Block(block_start, block_header, "V5_16_BIT_IQ_DATA");  

        break;
      default :
        // update block type processing log
          block_type_count[0] = block_type_count[0] + 1;

        // output message 
          output_log << "\n\t\t\"Warning\": \"Block not recognized. Type: " << block_header[2] << " at position: 0x" << hex << block_start-memblock << "\",";

          mem_cursor = mem_cursor + block_header[1] + BLOCK_TRAILER_SIZE;
    }

    return mem_cursor;
}

// structure to organize the active channel information. Channel divided into core and edges. The core segment is always present when a signal is detected, the segments between the core and the edges may be present or not.
  struct channel {
    // True if there is an active channel
      bool status;
    // Position of the begining of the active channel on the vector 
      long edge_begin; 
    // Position of the begining of the core of the active channel on the vector 
      long core_begin; 
    // Position of the end of the core of the active channel on the vector
      long core_end;
    // Position of the end of the active channel on the vector
      long edge_end;

    // null constructor    
      channel() : status(false), edge_begin(LONG_MIN), core_begin(LONG_MAX), core_end(LONG_MAX), edge_end(LONG_MIN) {}

    // reset to initialize new channel search
    void reset(void) {
      status = false;
      edge_begin = LONG_MIN;
      core_begin = LONG_MAX;
      core_end = LONG_MAX;
      edge_end = LONG_MIN;
    } 

  // update channel information with data from detected flags. Need to be initialized first with set method. No validation performed.
    void set(detection_flag *crossing_flag) {
      status = true;
      edge_begin = crossing_flag->segment_begin_position;
      core_begin = crossing_flag->segment_begin_position;
      core_end = crossing_flag->trace_position;
      edge_end = crossing_flag->trace_position;
    }

  // update channel information with data from detected flags. Need to be initialized first with set method. No validation performed.
    void update(detection_flag *crossing_flag) {
      // edge_begin should be the minimum position between all detected flags for the signal start (always move to the right, lower frequencies)
        if ( edge_begin > crossing_flag->segment_begin_position ) edge_begin = crossing_flag->segment_begin_position;

      // core_begin should be the maximum position between all detected flags for the signal start (always move to the left, higher frequencies)
        if ( core_begin < crossing_flag->segment_begin_position ) {
          // Update core begin if new begin still smaller than the core end
          if ( crossing_flag->segment_begin_position < core_end ) core_begin = crossing_flag->segment_begin_position;
        }

      // core_end should be the minumum position between all detected flags for the signal end (always move to the right, lower frequencies)
        if ( core_end > crossing_flag->trace_position ) {
          // Update core end if new end still larger than the core begin
          if ( crossing_flag->trace_position > core_begin ) core_end = crossing_flag->trace_position;
        }

      // edge_end should be the maximum position between all detected flags for the signal end (always move to the left, higher frequencies)
        if ( edge_end < crossing_flag->trace_position ) edge_end = crossing_flag->trace_position;
    }
  };

// vector that stores the detected channel information. Channel flags index is the same as the code index on the "channel list"
  vector<channel> channel_flags;

// perform feature extration for detected channels.
void channel_processing(void) {

    // numeric interator is used. Not used vector interator since multiple elements will be accessed at the same iteration
      int it;
    
    // Stores the detected channel information
      channel_status last_channel_status;

    // variables used for processing inside the loop;
      double density = 0.0;
      long channel_start_index;
      long channel_end_index;
      float *trace_iterator;
      float *trace_end_pointer;
      int frequency_index;
      int level_index;

    // create a vector with existing stream identifiers. Can not use iterator over stream since the number of elements on the stream unordered_map will be changed
      vector<double> stream_id_list;
      for (auto& one_stream: stream) {
        stream_id_list.push_back(one_stream.first);
      }

    // loop through all streams that exist prior to the channel processing
      for (auto & stream_id: stream_id_list) {

        // compute sample rate statistics looping through all traces on the current stream
          normal sample_rate_period;
          for ( it = 1; it < stream[stream_id].number_time_bins; it++ ) {
            sample_rate_period.add_element(((double)stream[stream_id].posix_time[it] - (double)stream[stream_id].posix_time[it-1]) + (((double)stream[stream_id].nanosecond_time[it]-(double)stream[stream_id].nanosecond_time[it-1])/NANOSECONDS_IN_SECOND));
          }

        // sort the threshold_crossing vector
          sort( stream[stream_id].threshold_crossing.begin( ), stream[stream_id].threshold_crossing.end( ));

        // reset iterator variables for the next stream
          it = 0;

          channel_flags.push_back(channel());

        // The following flag consolidation works by merging detected carriers that present intersepting cores. This will limit the ability to detect and differentiate carriers that share the same band but have different spectrum characteristics. e.g. Unauthorized user or interference over a channel will be interpreted as the channel.
        // loop through all flags crossings and consolidate flags into channel markers (flags)
          while ( it < stream[stream_id].threshold_crossing.size() ) {
            // if the crossing type is true, meaning it mark the end of a signal segment                            
              if  ( stream[stream_id].threshold_crossing.at(it).type ) {
                // if there is an active channel search
                  if (channel_flags.back().status) {
                    // if flag mark a segment that does not intercepts the active channel being processed (flag begin after the channel end)
                      if ( stream[stream_id].threshold_crossing.at(it).segment_begin_position > channel_flags.back().edge_end ) {
                        // create a new element at the end of the vector 
                          channel_flags.push_back(channel());
                        // start a new channel search using the newly created element
                          channel_flags.back().set(&stream[stream_id].threshold_crossing.at(it));
                      }
                      // if there is interception to the channel being processed
                      else {
                        // update channel information
                          channel_flags.back().update(&stream[stream_id].threshold_crossing.at(it));
                      }
                  }
                  // if there is no active channel ( only occurs at the beginning of the analysis)
                  else {
                    // start a new active channel
                      channel_flags.back().set(&stream[stream_id].threshold_crossing.at(it));
                  }
              }
            it++;
          }

        // TODO: Merge detected channels flags with user defined channels loaded from the database. Delete detected channels when the detected channel core includes a user defined core. Alternative set a flag on the database to control how merging will be conducted

        // loop through all channel markers (flags) and compute the corresponding channel data
          for (auto& one_channel: channel_flags) {        
            // Allocate space for the new channel data
              channel_list.push_back(channel_data(stream[stream_id].frequency[one_channel.edge_begin], // start frequency
                                                  stream[stream_id].frequency[one_channel.edge_end], // stop frequency
                                                  (uint32_t)((one_channel.edge_end-one_channel.edge_begin)+1), // number of frequency bins
                                                  DEFAULT_LEVEL_STEP , // number of level bins. Original range is 255 bins, from the reference level down to 127.5dB below. This variability could increase due to application of the antenna factor, an added element that varies with frequency. Still, data resolution is not increased and thus, actual values could be approximated with negligible error by using just 1 decimal place, meaning that doubling the current resolution is enough. i.e. from 255 bins for a representation of 0.5dB to 512 bins for a representation of 0.1dB.
                                                  stream[stream_id].level_min,
                                                  stream[stream_id].level_max));

            // updata the channel information on the new item of the channel list, using information from the original stream to which association is later lost
              channel_list.back().core_initial_frequency = stream[stream_id].frequency[one_channel.core_begin];
              channel_list.back().core_final_frequency = stream[stream_id].frequency[one_channel.core_end];

            // create the stream to store the channel information
              timespec reference_clock;
              reference_clock.tv_sec = stream[stream_id].posix_time[0];
              reference_clock.tv_nsec = stream[stream_id].nanosecond_time[0];

              stream[channel_list.back().stream_code] = trace_data(stream[stream_id].number_of_loops,
                                                                   stream[stream_id].sample_duration,
                                                                   channel_list.back().number_frequency_bins,
                                                                   stream[stream_id].number_time_bins,
                                                                   channel_list.back().start_frequency,
                                                                   channel_list.back().stop_frequency,
                                                                   reference_clock,
                                                                   false);


            // Register the channel vector index on stream for later reference
              stream[channel_list.back().stream_code].channel_code = channel_list.size()-1;

            // Update the detection threshold information on the channel stream
              stream[channel_list.back().stream_code].relative_thershold_value = stream_sweep[stream_id].detection_threshold; 

            // Set the values on the frequency axis on the channel data structure, copying the values from the corresponding 
              memcpy ( channel_list.back().frequency_axis, &stream[stream_id].frequency[one_channel.edge_begin], (sizeof(double)*channel_list.back().number_frequency_bins) );

            // clear vector that store the changes on channel state
              last_channel_status.set(UNKNOWN,stream[stream_id].posix_time[0],stream[stream_id].nanosecond_time[0]);

            // loop through all traces
              for ( it = 0; it < stream[stream_id].number_time_bins; it++ ) { 
                // compute density for the channel core
                  channel_end_index = ((long)it*(long)stream[stream_id].number_frequency_bins)+one_channel.core_end;
                  channel_start_index = ((long)it*(long)stream[stream_id].number_frequency_bins)+one_channel.core_begin;
                  density = (stream[stream_id].spectrogram_sum[channel_end_index] - stream[stream_id].spectrogram_sum[channel_start_index])/(double)(channel_end_index-channel_start_index);
                
                // store the density information with associated timestamp
                  channel_list.back().core_level_density.push_back((float)density);
                  channel_list.back().posix_time_for_density.push_back(stream[stream_id].posix_time[it]);
                  channel_list.back().nanosecond_time_for_density.push_back(stream[stream_id].nanosecond_time[it]);

                // if channel is active. i.e. density of the core is above the noise reference of the trace with added detection level
                  if ( density > stream[stream_id].noise_reference[it] + stream_sweep[stream_id].detection_threshold ) {

                      // move channel data from the main stream spectrogram to the channel stream, updating the channel data information
                        stream[stream_id].move_channel_trace(&stream[channel_list.back().stream_code],&channel_list.back(),it,one_channel.edge_begin,one_channel.edge_end);

                      // If there is a change on the channel state?
                        if ( last_channel_status.status != BUSY ) {
                          // Update the time information for when the channel become busy 
                            channel_list.back().ON_event_Posix.push_back(stream[stream_id].posix_time[it]);
                            channel_list.back().ON_event_Nanosecond.push_back(stream[stream_id].nanosecond_time[it]);

                          // Update the duration information for the free time of the channel
                            channel_list.back().OFF_event_duration.push_back( ((double)stream[stream_id].posix_time[it]-(double)last_channel_status.posix_time) + (((double)stream[stream_id].nanosecond_time[it]-(double)last_channel_status.nanosecond_time)/NANOSECONDS_IN_SECOND) );

                          // store the new status as the reference for the last status
                            last_channel_status.set(BUSY,stream[stream_id].posix_time[it],stream[stream_id].nanosecond_time[it]);  
                        }
                  }
                // else, if channel is inactive.
                  else {
                    // If there is a change on the channel state?
                      if ( last_channel_status.status != FREE ) {
                        // Update the time information for when the channel become free
                          channel_list.back().OFF_event_Posix.push_back(stream[stream_id].posix_time[it]);
                          channel_list.back().OFF_event_Nanosecond.push_back(stream[stream_id].nanosecond_time[it]);
                        
                        // Update the duration information for the busy time of the channel
                          channel_list.back().ON_event_duration.push_back( ((double)stream[stream_id].posix_time[it]-(double)last_channel_status.posix_time) + (((double)stream[stream_id].nanosecond_time[it]-(double)last_channel_status.nanosecond_time)/NANOSECONDS_IN_SECOND) );

                        // store the new status as the reference for the last status
                          last_channel_status.set(FREE,stream[stream_id].posix_time[it],stream[stream_id].nanosecond_time[it]);  
                      }                    
                  }
              }

            // set time for the begin status.
              channel_list.back().begin_status.set(UNKNOWN, stream[stream_id].posix_time[0], stream[stream_id].nanosecond_time[0]);

            // set time and status itself for end status.
              channel_list.back().end_status.set(last_channel_status.status, stream[stream_id].posix_time[stream[stream_id].number_time_bins-1], stream[stream_id].nanosecond_time[stream[stream_id].number_time_bins-1]);

            // count the number of channel status transitions
              int number_of_transitions = channel_list.back().ON_event_Posix.size() + channel_list.back().OFF_event_Posix.size() - 1;

            // process the channel information according to the number of transitions, handling cases for 0, 1 or many
              switch (number_of_transitions) {
                // if there is no transition, channel remains or or off during all time
                  case 0 : {
                    // The initial status is the same as the last status
                      channel_list.back().begin_status.status = last_channel_status.status;
                    
                    // The initial duration is equal to the file duration
                      channel_list.back().begin_status.duration = ((double)channel_list.back().end_status.posix_time - (double)channel_list.back().begin_status.posix_time) + (((double)channel_list.back().end_status.nanosecond_time - (double)channel_list.back().begin_status.nanosecond_time)/NANOSECONDS_IN_SECOND);
                    
                    // The end duration is 0, to avoid adding twice the duration when considering multiple files
                      channel_list.back().end_status.duration = 0.0;
                      
                    // If the status is ON
                      if ( last_channel_status.status == BUSY ) {
                        // Clear vectors set at the initial status transition from UNKNOWN to BUSY
                          channel_list.back().ON_event_Posix.clear();
                          channel_list.back().ON_event_Nanosecond.clear();
                          channel_list.back().OFF_event_duration.clear();
                      }
                    // else, if the status is OFF
                      else {
                        // Clear vectors set at the initial status transition from UNKNOWN to FREE
                          channel_list.back().OFF_event_Posix.clear();
                          channel_list.back().OFF_event_Nanosecond.clear();
                          channel_list.back().ON_event_duration.clear();
                      }
                  }
                  break;

                // if there is one transition, 
                  case 1 : {
                    // The initial status is the opposite of the last status
                      channel_list.back().begin_status.status = !last_channel_status.status;

                    // The initial duration is equal to the time until the last status time
                      channel_list.back().begin_status.duration = ((double)last_channel_status.posix_time - (double)channel_list.back().begin_status.posix_time) + (((double)last_channel_status.nanosecond_time - (double)channel_list.back().begin_status.nanosecond_time)/NANOSECONDS_IN_SECOND);

                    // The end duration is equal to the time from the last status time until the end
                      channel_list.back().end_status.duration = ((double)channel_list.back().end_status.posix_time - (double)last_channel_status.posix_time) + (((double)channel_list.back().end_status.nanosecond_time - (double)last_channel_status.nanosecond_time)/NANOSECONDS_IN_SECOND);

                    // Clear all transition vectors, since there is not a conclusive duration measurement within the file
                      channel_list.back().ON_event_Posix.clear();
                      channel_list.back().ON_event_Nanosecond.clear();
                      channel_list.back().OFF_event_duration.clear();
                      channel_list.back().OFF_event_Posix.clear();
                      channel_list.back().OFF_event_Nanosecond.clear();
                      channel_list.back().ON_event_duration.clear();
                  }
                  break;
                
                // if there are many transitions,
                  default : {
                    // If the first event corresponds to an ON event. Compare the time for the initial events and check if it matches the initial trace time to infer if the initial status was ON 
                      if ( (channel_list.back().ON_event_Posix[0] == stream[stream_id].posix_time[0]) && (channel_list.back().ON_event_Nanosecond[0] == stream[stream_id].nanosecond_time[0]) ) {
                        // remove the first element of the ON event list, since it marks the initial status but has no valid duration. 
                          channel_list.back().ON_event_Posix.erase(channel_list.back().ON_event_Posix.begin());
                          channel_list.back().ON_event_Nanosecond.erase(channel_list.back().ON_event_Nanosecond.begin());
                          channel_list.back().OFF_event_duration.erase(channel_list.back().OFF_event_duration.begin());

                        // set the begin status 
                          channel_list.back().begin_status.status = BUSY;

                        // if there was a transition to 
                        // get the duration until the end of the initial ON condition
                          channel_list.back().begin_status.duration = channel_list.back().ON_event_duration.front();
                        // remove the initial duration from the duration vector, since it is not valid because might be incomplete
                          channel_list.back().ON_event_duration.erase(channel_list.back().ON_event_duration.begin());
                      }
                      // Else, if it corresponds to an off event
                      else {
                        // remove the first element of the off event list, since it marks the initial status but has no valid duration. 
                          channel_list.back().OFF_event_Posix.erase(channel_list.back().OFF_event_Posix.begin());
                          channel_list.back().OFF_event_Nanosecond.erase(channel_list.back().OFF_event_Nanosecond.begin());
                          channel_list.back().ON_event_duration.erase(channel_list.back().ON_event_duration.begin());

                        // set the begin status 
                          channel_list.back().begin_status.status = FREE;

                        // get the duration until the end of the initial OFF condition
                          channel_list.back().begin_status.duration = channel_list.back().OFF_event_duration.front();
                        // remove the initial duration from the duration vector, since it is not valid because might be incomplete
                          channel_list.back().OFF_event_duration.erase(channel_list.back().OFF_event_duration.begin());
                      }
                    
                    // set the duration of the end status taking into consideration the last status change reference
                      channel_list.back().end_status.duration = ((double)stream[stream_id].posix_time[stream[stream_id].number_time_bins-1]-(double)last_channel_status.posix_time) + (((double)stream[stream_id].nanosecond_time[stream[stream_id].number_time_bins-1]-(double)last_channel_status.nanosecond_time)/NANOSECONDS_IN_SECOND);

                    // if the last channel status is busy
                      if ( last_channel_status.status == BUSY ) {
                        // delete the time reference for the last status change on the ON vectors
                          channel_list.back().ON_event_Posix.pop_back();
                          channel_list.back().ON_event_Nanosecond.pop_back();
                      }
                    // else, if the last status is free
                      else {
                        // delete the time reference for the last status change on the OFF vectors
                          channel_list.back().OFF_event_Posix.pop_back();
                          channel_list.back().OFF_event_Nanosecond.pop_back();
                      }
                  }
              }

            // Update the channel sample rate period in accordance with the previously computed value
              channel_list.back().sample_rate_period = sample_rate_period;

            // TODO: Release unused memory from the 

          }

        // Reprocess the source trace data for the current stream, computing noise information for the band
        
        // Allocate space to store noise information as the last channel
          channel_list.push_back(channel_data(stream[stream_id].frequency[0], // start frequency
                                              stream[stream_id].frequency[stream[stream_id].number_frequency_bins-1], // stop frequency
                                              stream[stream_id].number_frequency_bins, // number of frequency bins
                                              DEFAULT_LEVEL_STEP , // size of the level step, to be used to compute the number of level bins based on the maximum and minimum values
                                              stream[stream_id].level_min,
                                              stream[stream_id].level_max));
        
        // Update the channel sample rate period in accordance with the previously computed value
          channel_list.back().sample_rate_period = sample_rate_period;

        // Set the values on the frequency axis on the channel data structure, copying the values from the corresponding 
          memcpy ( channel_list.back().frequency_axis, &stream[stream_id].frequency[0], (sizeof(double)*channel_list.back().number_frequency_bins) );

        // Register the channel vector index on stream for later reference
          stream[stream_id].channel_code = channel_list.size()-1;

        
        // Consolidate noise information on the trace data and on the channel list
          stream[stream_id].abstract_noise(stream_id,&channel_list.back());

      }
}

// stores trace_data objects into HDF5 format
int store_hdf5(string start_time_string, string file_name) {

  // create simplified structure to store the positional data on the HDF5 file
    struct simple_site {
      // latitude average measurement
        double latitude;
      // longitude average measurement
        double longitude; 
      // altitude average measurement 
        double altitude; 
      // posix timestamp for the last measurement
        uint32_t timestamp_coarse;
      // nanosecond timestamp for the last measurement
        uint32_t timestamp_fine;

      // null constructor    
        simple_site() : latitude(0.0), longitude(0.0), altitude(0.0), timestamp_coarse(0), timestamp_fine(0) {}

      // reset to initialize new channel search
        simple_site(position site_data) {
          latitude = site_data.latitude.mean_value ;
          longitude = site_data.longitude.mean_value;
          altitude = site_data.altitude.mean_value;
          timestamp_coarse = site_data.stop_clock.tv_sec;
          timestamp_fine = site_data.stop_clock.tv_nsec;
        } 
    };

  // Try block to detect exceptions raised by any of the calls inside it
    try {
	    // Turn off the auto-printing to avoid wrong output formatting
	      Exception::dontPrint();

	    // Create a new file using the default property lists. HDF5_filename.str()
	      H5File file(file_name, H5F_ACC_TRUNC);

      // Modify dataset creation property to enable chunking. Chunking is necessary in order to be able to later extend the dataset, appending new data
        DSetCreatPropList  plist_2D;
        DSetCreatPropList  plist_1D;
        
      // Set chunk dimensions for 1 dimension qnd 2 dimensions datasets
        hsize_t dims_2D[RANK_2D];
        hsize_t dims_1D[RANK_1D];
	      plist_2D.setChunk(RANK_2D, chunk_dims_2D);
        plist_1D.setChunk(RANK_1D, chunk_dims_1D);

	    // Set ZLIB (DEFLATE) Compression using level 6.
	      plist_2D.setDeflate(HDF5_COMPRESSION_LEVEL);
        plist_1D.setDeflate(HDF5_COMPRESSION_LEVEL);

      // Create datatypes that will be used on multiple groups/attributes
        // create a variable string type to be used as reference type
          StrType h5_string(0, H5T_VARIABLE);

        // create the compound data type used to store normally distributed data statistics
          CompType h5_statistics( sizeof(normal) );
          h5_statistics.insertMember( MEAN_MEMBER, HOFFSET(normal, mean_value), PredType::NATIVE_DOUBLE);
          h5_statistics.insertMember( STANDARD_DEVIATION_MEMBER, HOFFSET(normal, std_value), PredType::NATIVE_DOUBLE);
          h5_statistics.insertMember( NUMBER_OF_SAMPLES_MEMBER, HOFFSET(normal, count), PredType::NATIVE_INT);
          h5_statistics.insertMember( SUM_MEMBER, HOFFSET(normal, sum), PredType::NATIVE_DOUBLE);
          h5_statistics.insertMember( SUM_OF_SQUARES_MEMBER, HOFFSET(normal, sum_squares), PredType::NATIVE_DOUBLE);

        // Create variables that will be used for multiple attributes
          dims_1D[0] = 1;
          DataSpace single_attribute_dataspace = DataSpace(RANK_1D, dims_1D);
          Attribute attribute;

        // Crerate attributes associated with the file root
          attribute = file.createAttribute(STANDARD_ATTRIBUTE, h5_string, single_attribute_dataspace);
          attribute.write(h5_string,STANDARD);

          attribute = file.createAttribute(CLASS_ATTRIBUTE, h5_string, single_attribute_dataspace);
          attribute.write(h5_string,FILETYPE_CLASS);

          attribute = file.createAttribute(RESPONSIBLE_ATTRIBUTE, h5_string, single_attribute_dataspace);
          attribute.write(h5_string,RESPONSIBLE);

          attribute = file.createAttribute(CREATION_TIMESTAMP_ATTRIBUTE, h5_string, single_attribute_dataspace);
          attribute.write(h5_string,start_time_string);

      // store the site geolocation information 
        {
          // create the compound data type used to store the site geolocation
            CompType h5_site_geolocation( sizeof(simple_site) ); 
            h5_site_geolocation.insertMember( LATITUDE_MEMBER, HOFFSET(simple_site, latitude), PredType::NATIVE_DOUBLE);
            h5_site_geolocation.insertMember( LONGITUDE_MEMBER, HOFFSET(simple_site, longitude), PredType::NATIVE_DOUBLE);
            h5_site_geolocation.insertMember( ALTITUDE_MEMBER, HOFFSET(simple_site, altitude), PredType::NATIVE_DOUBLE);
            h5_site_geolocation.insertMember( TIMESTAMP_COARSE_MEMBER, HOFFSET(simple_site, timestamp_coarse), PredType::NATIVE_UINT32);
            h5_site_geolocation.insertMember( TIMESTAMP_FINE_MEMBER, HOFFSET(simple_site, timestamp_fine), PredType::NATIVE_UINT32);

          // construct object with actual values to be strored;
            simple_site site_geolocation(site);

          // Create dataspace for the dataset
            dims_1D[0] = 1;
            DataSpace site_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);
          
          // Create the dataset
            DataSet site_dataset = file.createDataSet(SITE_GEOLOCATION_DATASET, h5_site_geolocation, site_dataspace, plist_1D);

          // Write dataset to file
            site_dataset.write(&site_geolocation, h5_site_geolocation);

          // Create attributes associated with the site dataset
            attribute = site_dataset.createAttribute(STANDARD_ATTRIBUTE, h5_string, single_attribute_dataspace);
            attribute.write(h5_string,STANDARD);

            attribute = site_dataset.createAttribute(CLASS_ATTRIBUTE, h5_string, single_attribute_dataspace);
            attribute.write(h5_string,GEOLOCATION_CLASS);

            attribute = site_dataset.createAttribute(GEODETIC_DATUM_ATTRIBUTE, h5_string, single_attribute_dataspace);
            attribute.write(h5_string,CRFS_BIN_DEFAULT_GEODETIC_DATUM);

            attribute = site_dataset.createAttribute(LATITUDE_STATISTICS_ATTRIBUTE, h5_statistics, single_attribute_dataspace);
            attribute.write(h5_statistics,&site.latitude);

            attribute = site_dataset.createAttribute(LONGITUDE_STATISTICS_ATTRIBUTE, h5_statistics, single_attribute_dataspace);
            attribute.write(h5_statistics,&site.longitude);

            attribute = site_dataset.createAttribute(ALTITUDE_STATISTICS_ATTRIBUTE, h5_statistics, single_attribute_dataspace);
            attribute.write(h5_statistics,&site.altitude);            
        };

      // Include equipment data on the log
        equipment.set_to_log(site.start_clock,&event_log);
      
      // sort the event log prior to storing
        sort( event_log.begin(), event_log.end() );

      // store the logbook information
        {
          // create the compound data type used to store the logbook
            CompType h5_logbook( sizeof(event_item) ); 
            h5_logbook.insertMember( TIMESTAMP_COARSE_MEMBER, HOFFSET(event_item, posix_time), PredType::NATIVE_UINT32);
            h5_logbook.insertMember( TIMESTAMP_FINE_MEMBER, HOFFSET(event_item, nanosecond_time), PredType::NATIVE_UINT32);
            h5_logbook.insertMember( ENTRY_TYPE_MEMBER, HOFFSET(event_item, type), h5_string);
            h5_logbook.insertMember( ENTRY_VALUE_MEMBER, HOFFSET(event_item, value), h5_string);

          // Create dataspace
            dims_1D[0] = event_log.size();
            DataSpace logbook_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);
          
          // Create the dataset
            DataSet logbook_dataset = DataSet(file.createDataSet(LOGBOOK_DATASET, h5_logbook, logbook_dataspace, plist_1D));

          // Write data to file
            logbook_dataset.write(&event_log[0], h5_logbook);
        };

      // create variables for the base groups and reference group where data stream will be ocurring 
        Group em_spectrum_action_group;
        Group level_profile_action_group;
        Group activity_profile_action_group;
        Group group_sweep;
        Group group_noise;
        Group group_channel; 

// TODO: Handle the situation when there are no channels detected and all is noise.       
      // if no detection mode was selected
        if ( detection_mode == NO_DETECTION_OPTION ) {
          // All trace data will be stored within the frequency sweep group;
            group_sweep = file.createGroup(FREQUENCY_SWEEP_DATA_GROUP);
        }
      // if a detection mode was selected,
        else {
          // Set the name for the level and activity profile
            group_noise = file.createGroup(NOISE_DATA_GROUP);
            group_channel = file.createGroup(CHANNEL_DATA_GROUP);
        }

      // declare string variables to be used to create different names for each stream 
        string stream_code_string;
        string HDF5_group_name_em_spectrum;
        string HDF5_group_name_level_profile;
        string HDF5_group_name_activity_profile;

      // declare variable for the noise for each channel
        unordered_map<double,normal> noise_reference_statistics;

      // loop through all streams storing the corresponding datasets
        for (auto& one_stream: stream) {

          // Get the stream number code as a suffix to the dataset name 
            stream_code_string =  to_string((int)one_stream.first);
            HDF5_group_name_em_spectrum = EM_SPECTRUM_DATA_GROUP + stream_code_string;

          // if no detection mode was selected
            if ( detection_mode == NO_DETECTION_OPTION ) {
              // All trace data will be stored within the frequency sweep group;
              em_spectrum_action_group = group_sweep.createGroup(HDF5_group_name_em_spectrum);
            }
          // if a detection mode was selected,
            else {
              // set the name for the level profile group
                HDF5_group_name_level_profile = LEVEL_PROFILE_DATA_GROUP + stream_code_string;

              // if the stream ID code less than the minimum channel acceptable code, the stream refers to noise data 
                if (one_stream.first < MINIMUM_CHANNEL_THREAD_ID) {
                  // set the groups where the data will be stored within the noise group
                    em_spectrum_action_group = group_noise.createGroup(HDF5_group_name_em_spectrum);
                    level_profile_action_group = group_noise.createGroup(HDF5_group_name_level_profile);
                } 
              // else, the stream refers to channel data
                else {
                  // set the groups where the data will be stored within the channel group
                    em_spectrum_action_group = group_channel.createGroup(HDF5_group_name_em_spectrum);
                    level_profile_action_group = group_channel.createGroup(HDF5_group_name_level_profile);
                }
            }            

          // store the EM spectrum
            {
              // Store group attributes associated with the EM spectrum group
                attribute = em_spectrum_action_group.createAttribute(STANDARD_ATTRIBUTE, h5_string, single_attribute_dataspace);
                attribute.write(h5_string,STANDARD);

                attribute = em_spectrum_action_group.createAttribute(CLASS_ATTRIBUTE, h5_string, single_attribute_dataspace);
                attribute.write(h5_string,SPECTROGRAM_CLASS);

                attribute = em_spectrum_action_group.createAttribute(NUMBER_OF_LOOPS_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                attribute.write(PredType::NATIVE_UINT32,&one_stream.second.number_of_loops);

                attribute = em_spectrum_action_group.createAttribute(SAMPLE_DURATION_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.sample_duration);

              // Create the spectrogram dataset.
                // create data space
                  dims_2D[0] = one_stream.second.number_time_bins;
                  dims_2D[1] = one_stream.second.number_frequency_bins;
                  DataSpace spectrogram_dataspace = DataSpace(RANK_2D, dims_2D, maxdims_2D);

                // Create and write dataset
                  DataSet spectrogram_dataset = DataSet(em_spectrum_action_group.createDataSet(SPECTROGRAM_DATASET, PredType::NATIVE_FLOAT, spectrogram_dataspace, plist_2D));
                  spectrogram_dataset.write(one_stream.second.spectrogram, PredType::NATIVE_FLOAT);

                // Create attributes associated with the dataset
                  attribute = spectrogram_dataset.createAttribute(MEASUREMENT_UNIT_ATTRIBUTE, h5_string, single_attribute_dataspace);
                  attribute.write(h5_string,output_unit_label);

                  attribute = spectrogram_dataset.createAttribute(MAXIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_max);

                  attribute = spectrogram_dataset.createAttribute(MINIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_min);

              // create dataset for time axis data
                // create data space
                  dims_1D[0] = one_stream.second.number_time_bins;
                  DataSpace time_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);              

                // Create and write timestamp datasets
                  DataSet posix_dataset = DataSet(em_spectrum_action_group.createDataSet(TIMESTAMP_COARSE_DATASET, PredType::NATIVE_UINT32, time_dataspace, plist_1D));
                  DataSet nanosecond_dataset = DataSet(em_spectrum_action_group.createDataSet(TIMESTAMP_FINE_DATASET, PredType::NATIVE_UINT32, time_dataspace, plist_1D));                

                  posix_dataset.write(one_stream.second.posix_time,PredType::NATIVE_UINT32);
                  nanosecond_dataset.write(one_stream.second.nanosecond_time,PredType::NATIVE_UINT32);

                // Create attributes associated with the dataset
                  attribute = posix_dataset.createAttribute(START_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_UINT32,&one_stream.second.posix_time[0]);

                  attribute = posix_dataset.createAttribute(STOP_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_UINT32,&one_stream.second.posix_time[one_stream.second.number_time_bins-1]);

                  attribute = nanosecond_dataset.createAttribute(START_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_UINT32,&one_stream.second.nanosecond_time[0]);

                  attribute = nanosecond_dataset.createAttribute(STOP_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_UINT32,&one_stream.second.nanosecond_time[one_stream.second.number_time_bins-1]);

              // create dataset for frequency axis data
                // create data space
                  dims_1D[0] = one_stream.second.number_frequency_bins;              
                  DataSpace frequency_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                // Create and write dataset
                  DataSet frequency_dataset = DataSet(em_spectrum_action_group.createDataSet(FREQUENCY_DATASET, PredType::NATIVE_DOUBLE, frequency_dataspace, plist_1D));
                  frequency_dataset.write(one_stream.second.frequency,PredType::NATIVE_DOUBLE);

                // Create attributes associated with the dataset
                  attribute = frequency_dataset.createAttribute(FILTER_BANDWIDTH_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency_step);

                  attribute = frequency_dataset.createAttribute(INITIAL_FREQUENCY_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency[0]);

                  attribute = frequency_dataset.createAttribute(FINAL_FREQUENCY_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                  attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency[one_stream.second.number_frequency_bins-1]);

              // close EM spectrum group
                em_spectrum_action_group.close();
            };

          // If there was selected a detection mode and thus the stream referes to a simple spectrum sweep
            if ( detection_mode == NO_DETECTION_OPTION ) {
              // Nothing else to do, just close the spectrum sweep group. this group will not exist if channel detection was used
                group_sweep.close();
            }
            // else, the stream refers to channel and noise data
            else {
              // create a pointer to the channel associated with the current stream
                channel_data *current_channel = &channel_list[one_stream.second.channel_code];
                channel *current_channel_limits = &channel_flags[one_stream.second.channel_code];

              // store the activity profile dataset, this is available for both the noise and the channel data.
                {
                  // Store group attributes associated with the level profile group
                    attribute = level_profile_action_group.createAttribute(STANDARD_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,STANDARD);

                    attribute = level_profile_action_group.createAttribute(CLASS_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,LEVEL_PROFILE_CLASS);

                    attribute = level_profile_action_group.createAttribute(START_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&one_stream.second.posix_time[0]);

                    attribute = level_profile_action_group.createAttribute(STOP_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&one_stream.second.posix_time[one_stream.second.number_time_bins-1]);

                    attribute = level_profile_action_group.createAttribute(START_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&one_stream.second.nanosecond_time[0]);

                    attribute = level_profile_action_group.createAttribute(STOP_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&one_stream.second.nanosecond_time[one_stream.second.number_time_bins-1]);

                  // compute new size for the power signature and level axis, cutting final rows that do not contain data
                    uint32_t level_axis_size = (one_stream.second.level_min_index-one_stream.second.level_max_index)+1;

                  // compute new initial pointer for the power signature, cutting the final rows that do not contain data
                    uint16_t *power_signature_begin = &current_channel->power_signature[one_stream.second.level_max_index*current_channel->number_frequency_bins];

                  // Create the data space for the level profile dataset.
                    dims_2D[0] = level_axis_size;
                    dims_2D[1] = current_channel->number_frequency_bins;
                    DataSpace level_profile_dataspace = DataSpace(RANK_2D, dims_2D, maxdims_2D);

                  // create and write the dataset for the level profile
                    DataSet level_profile_dataset = DataSet(level_profile_action_group.createDataSet(LEVEL_PROFILE_DATASET, PredType::NATIVE_UINT16, level_profile_dataspace, plist_2D));
                    level_profile_dataset.write(power_signature_begin, PredType::NATIVE_UINT16);

                  // write attributes associated with the level profile
                    attribute = level_profile_dataset.createAttribute(MEASUREMENT_UNIT_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,output_unit_label);

                    attribute = level_profile_dataset.createAttribute(LEVEL_STEP_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->level_step);

                    attribute = level_profile_dataset.createAttribute(NUMBER_OF_PROFILE_TRACES_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&current_channel->number_of_samples);

                    attribute = level_profile_dataset.createAttribute(MAXIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_max);

                    attribute = level_profile_dataset.createAttribute(MINIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_min);

                  // compute new initial pointer for the level axis array, cutting initial rows that do not contain data
                    double *level_axis_begin = &current_channel->level_axis[one_stream.second.level_max_index];

                  // Create the data space for the level axis dataset.
                    dims_1D[0] = level_axis_size;
                    DataSpace level_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                  // create dataset and write the level axis data
                    DataSet level_dataset = DataSet(level_profile_action_group.createDataSet(LEVEL_DATASET, PredType::NATIVE_DOUBLE, level_dataspace, plist_1D));
                    level_dataset.write(level_axis_begin, PredType::NATIVE_DOUBLE);            

                  // write attributes associated with the level axis
                    attribute = level_dataset.createAttribute(MEASUREMENT_UNIT_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,output_unit_label);

                    attribute = level_dataset.createAttribute(LEVEL_STEP_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->level_step);

                    attribute = level_dataset.createAttribute(MAXIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_max);

                    attribute = level_dataset.createAttribute(MINIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.level_min);

                  // create dataset for the frequency axis data
                    dims_1D[0] = current_channel->number_frequency_bins;
                    DataSpace frequency_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                  // create dataset and write the frequency axis data
                    DataSet frequency_dataset = DataSet(level_profile_action_group.createDataSet(FREQUENCY_DATASET, PredType::NATIVE_DOUBLE, frequency_dataspace, plist_1D));
                    frequency_dataset.write(current_channel->frequency_axis, PredType::NATIVE_DOUBLE);

                  // write attributes associated with the frequency axis
                    attribute = frequency_dataset.createAttribute(FILTER_BANDWIDTH_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency_step);

                    attribute = frequency_dataset.createAttribute(INITIAL_FREQUENCY_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency[0]);

                    attribute = frequency_dataset.createAttribute(FINAL_FREQUENCY_ATTRIBUTE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.frequency[one_stream.second.number_frequency_bins-1]);
                };

              // if the stream ID code greater than the maximum acceptable code, the stream refers to channel data 
                if (one_stream.first > MINIMUM_CHANNEL_THREAD_ID) {

                  // store the additional information about level statistics within the channel on the level profile group
                    {
                      // create dataspacec for vectos with dimension equal to the number of frequency bins on the channel
                        dims_1D[0] = current_channel->number_frequency_bins;
                        DataSpace frequency_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                      // create dataset and write the level statistics data
                        DataSet level_statistics_dataset = DataSet(level_profile_action_group.createDataSet(LEVEL_STATISTICS_DATASET, h5_statistics, frequency_dataspace, plist_1D));
                        level_statistics_dataset.write(&current_channel->level_normal[0], h5_statistics);

                      // Create attributes associated with the dataset
                        attribute = level_statistics_dataset.createAttribute(MEASUREMENT_UNIT_ATTRIBUTE, h5_string, single_attribute_dataspace);
                        attribute.write(h5_string,output_unit_label);

                      // create dataspace for vector with dimension equal to the number of original time samples 
                        dims_1D[0] = current_channel->core_level_density.size() ;
                        DataSpace time_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);              

                      // create dataset and write the level density data
                        DataSet level_density_dataset = DataSet(level_profile_action_group.createDataSet(LEVEL_DENSITY_DATASET, PredType::NATIVE_FLOAT, time_dataspace, plist_1D));
                        level_density_dataset.write(&current_channel->core_level_density[0], PredType::NATIVE_FLOAT);

                      // Create attributes associated with the dataset
                        attribute = level_density_dataset.createAttribute(MEASUREMENT_UNIT_ATTRIBUTE, h5_string, single_attribute_dataspace);
                        attribute.write(h5_string,output_unit_label);

                        float minimum = current_channel->core_level_density[0];
                        float maximum = current_channel->core_level_density[0];
                        for (auto& one_level: current_channel->core_level_density) {
                            if (one_level < minimum) {
                                minimum = one_level;
                            }
                            else if (one_level > maximum) {
                                maximum = one_level;
                            };
                        };

                        attribute = level_density_dataset.createAttribute(MAXIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_FLOAT, single_attribute_dataspace);
                        attribute.write(PredType::NATIVE_FLOAT,&minimum);

                        attribute = level_density_dataset.createAttribute(MINIMUM_DETECTED_LEVEL_ATTRIBUTE, PredType::NATIVE_FLOAT, single_attribute_dataspace);
                        attribute.write(PredType::NATIVE_FLOAT,&maximum);

                      // TODO: CREATE HARD LINK TO TIME AXIS INSTEAD OF REPLICATING DATA
                      // create dataset for time axis data associated with the level density data
                        // Create and write timestamp datasets
                          DataSet posix_dataset = DataSet(level_profile_action_group.createDataSet(TIMESTAMP_COARSE_DATASET, PredType::NATIVE_UINT32, time_dataspace, plist_1D));
                          DataSet nanosecond_dataset = DataSet(level_profile_action_group.createDataSet(TIMESTAMP_FINE_DATASET, PredType::NATIVE_UINT32, time_dataspace, plist_1D));                

                          posix_dataset.write(&current_channel->posix_time_for_density[0],PredType::NATIVE_UINT32);
                          nanosecond_dataset.write(&current_channel->nanosecond_time_for_density[0],PredType::NATIVE_UINT32);

                        // Create attributes associated with the dataset
                          attribute = posix_dataset.createAttribute(START_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                          attribute.write(PredType::NATIVE_UINT32,&current_channel->posix_time_for_density[0]);

                          attribute = posix_dataset.createAttribute(STOP_TIME_COARSE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                          attribute.write(PredType::NATIVE_UINT32,&current_channel->posix_time_for_density.back());

                          attribute = nanosecond_dataset.createAttribute(START_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                          attribute.write(PredType::NATIVE_UINT32,&current_channel->nanosecond_time_for_density.back());

                          attribute = nanosecond_dataset.createAttribute(STOP_TIME_FINE_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                          attribute.write(PredType::NATIVE_UINT32,&one_stream.second.nanosecond_time[one_stream.second.number_time_bins-1]);
                    }

                  // Set the name for the level and activity profile
                    HDF5_group_name_activity_profile = ACTIVITY_PROFILE_DATA_GROUP + stream_code_string;
                  
                  // set the groups where the data will be stored within the channel group
                    activity_profile_action_group = group_channel.createGroup(HDF5_group_name_activity_profile);

                  // Store group attributes associated with the activity profile group
                    attribute = activity_profile_action_group.createAttribute(STANDARD_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,STANDARD);

                    attribute = activity_profile_action_group.createAttribute(CLASS_ATTRIBUTE, h5_string, single_attribute_dataspace);
                    attribute.write(h5_string,ACTIVITY_PROFILE_CLASS);

                    attribute = activity_profile_action_group.createAttribute(FIRST_TIMESTAMP_COARSE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&current_channel->begin_status.posix_time);

                    attribute = activity_profile_action_group.createAttribute(FIRST_TIMESTAMP_FINE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&current_channel->begin_status.nanosecond_time);

                    attribute = activity_profile_action_group.createAttribute(DURATION_UNTIL_FIRST, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->begin_status.duration);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_ACTIVE_AT_START_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    bool current_channel_status = current_channel->begin_status.is_active();
                    attribute.write(PredType::NATIVE_HBOOL,&current_channel_status);

                    attribute = activity_profile_action_group.createAttribute(LAST_TIMESTAMP_COARSE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&current_channel->end_status.posix_time);

                    attribute = activity_profile_action_group.createAttribute(LAST_TIMESTAMP_FINE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_UINT32,&current_channel->end_status.nanosecond_time);

                    attribute = activity_profile_action_group.createAttribute(DURATION_UNTIL_END, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->end_status.duration);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_ACTIVE_AT_STOP_ATTRIBUTE, PredType::NATIVE_UINT32, single_attribute_dataspace);
                    current_channel_status = current_channel->end_status.is_active();
                    attribute.write(PredType::NATIVE_HBOOL,&current_channel_status);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_EDGE_INITIAL_FREQUENCY, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->frequency_axis[0]);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_CORE_INITIAL_FREQUENCY, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->core_initial_frequency);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_CORE_FINAL_FREQUENCY, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->core_final_frequency);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_EDGE_FINAL_FREQUENCY, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->frequency_axis[current_channel->number_frequency_bins-1]);

                  // compute noise statistics to be used as reference on the channel data 
                      normal *noise_reference_pointer;
                    // test if the noise statistics were not already computed for the original stream associated with the current channel
                      if (noise_reference_statistics.find(current_channel->stream_code) == noise_reference_statistics.end()) {
                        // create the new entry on the noise_reference_statistics map
                          noise_reference_statistics[current_channel->stream_code] = normal();
                        // loop through all elements on the reference noise vector computing statistics as for a normal distribution
                          noise_reference_pointer =  &noise_reference_statistics[current_channel->stream_code];
                          for (auto& trace_noise: one_stream.second.noise_reference ) {
                            noise_reference_pointer->add_element(trace_noise);
                            // TODO: May be improved. Several assumptions are potentially wrong here. Each element is composed as the average of an unkonwn number of atomic elements, this implies that each element should not have an equivalent weight on the averaging process. Also the assumption of a normmal distribution, although not absurd, is not necessarily true. Even disconsidering such problems, the implementation itself could be improved since the 
                          }
                      } else {
                        noise_reference_pointer =  &noise_reference_statistics[current_channel->stream_code];
                      }

                    attribute = activity_profile_action_group.createAttribute(NOISE_REFERENCE_STATISTICS, h5_statistics, single_attribute_dataspace);
                    attribute.write(h5_statistics,noise_reference_pointer);

                    attribute = activity_profile_action_group.createAttribute(DETECTION_THRESHOLD, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&one_stream.second.relative_thershold_value);

                    attribute = activity_profile_action_group.createAttribute(AVERAGE_CHANNEL_SAMPLE_RATE, PredType::NATIVE_DOUBLE, single_attribute_dataspace);
                    attribute.write(PredType::NATIVE_DOUBLE,&current_channel->sample_rate_period.mean_value);

                    attribute = activity_profile_action_group.createAttribute(CHANNEL_SAMPLE_RATE_STATISTICS, h5_statistics, single_attribute_dataspace);
                    attribute.write(h5_statistics,&current_channel->sample_rate_period);

                  // If the channel presented activity switching to ON at least one time
                    if ( current_channel->ON_event_duration.size() > 0 ) {

                      // Create the data space for the ON events datasets.
                        dims_1D[0] = current_channel->ON_event_duration.size();
                        DataSpace activation_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                      // create dataset for ON event coarse timestamp
                        DataSet activation_timestamp_coarse_dataset = DataSet(activity_profile_action_group.createDataSet(ACTIVATION_EVENT_COARSE_TIMESTAMP_DATASET, PredType::NATIVE_UINT32, activation_dataspace, plist_1D));
                        activation_timestamp_coarse_dataset.write(&current_channel->ON_event_Posix[0], PredType::NATIVE_UINT32);

                      // create dataset for ON event fine timestamp
                        DataSet  activation_timestamp_fine_dataset = DataSet(activity_profile_action_group.createDataSet(ACTIVATION_EVENT_FINE_TIMESTAMP_DATASET, PredType::NATIVE_UINT32, activation_dataspace, plist_1D));
                        activation_timestamp_fine_dataset.write(&current_channel->ON_event_Nanosecond[0], PredType::NATIVE_UINT32);

                      // create dataset for ON event duration
                        DataSet activated_duration_dataset = DataSet(activity_profile_action_group.createDataSet(ACTIVATED_STATE_DURATION_DATASET, PredType::NATIVE_DOUBLE, activation_dataspace, plist_1D));
                        activated_duration_dataset.write(&current_channel->ON_event_duration[0], PredType::NATIVE_DOUBLE);
                    };

                  // If the channel presented activity switching to OFF at least one time
                    if ( current_channel->OFF_event_duration.size() > 0 ) {

                      // Create the data space for the OFF events datasets.
                        dims_1D[0] = current_channel->OFF_event_duration.size();
                        DataSpace deactivation_dataspace = DataSpace(RANK_1D, dims_1D, maxdims_1D);

                      // create dataset for OFF event coarse timestamp
                        DataSet deactivation_timestamp_coarse_dataset = DataSet(activity_profile_action_group.createDataSet(DEACTIVATION_EVENT_COARSE_TIMESTAMP_DATASET, PredType::NATIVE_UINT32, deactivation_dataspace, plist_1D));
                        deactivation_timestamp_coarse_dataset.write(&current_channel->OFF_event_Posix[0], PredType::NATIVE_UINT32);

                      // create dataset for OFF event fine timestamp
                        DataSet deactivation_timestamp_fine_dataset = DataSet(activity_profile_action_group.createDataSet(DEACTIVATION_EVENT_FINE_TIMESTAMP_DATASET, PredType::NATIVE_UINT32, deactivation_dataspace, plist_1D));
                        deactivation_timestamp_fine_dataset.write(&current_channel->OFF_event_Nanosecond[0], PredType::NATIVE_UINT32);

                      // create dataset for ON event duration
                        DataSet deactivated_duration_dataset = DataSet(activity_profile_action_group.createDataSet(DEACTIVATED_STATE_DURATION_DATASET, PredType::NATIVE_DOUBLE, deactivation_dataspace, plist_1D));
                        deactivated_duration_dataset.write(&current_channel->OFF_event_duration[0], PredType::NATIVE_DOUBLE);
                    };

                  // close activity profile group
                    activity_profile_action_group.close();
                } 

              // close level profile group
                level_profile_action_group.close();
            }     
        }

      // Close all objects and file.
        group_noise.close();
        group_channel.close();

        file.close();  
    }  // end of try block

  // catch failure caused by the H5File operations
    catch( DataSpaceIException error ) {
      // error.printErrorStack();
      output_log << "\n\t\t\"Error\": \"HDF5 Dataspace exception\"\n\t}\n}";
      return EXIT_FAILURE;
    }

  // catch failure caused by the H5File operations
    catch( AttributeIException error ) {
      // error.printErrorStack();
      output_log << "\n\t\t\"Error\": \"HDF5 Attribute exception\"\n\t}\n}";
      return EXIT_FAILURE;
    }

  // catch failure caused by the H5File operations
    catch( FileIException error ) {
      // error.printErrorStack();
      output_log << "\n\t\t\"Error\": \"HDF5 File access exception\"\n\t}\n}";
      return EXIT_FAILURE;
    }

  // catch failure caused by the DataSet operations
    catch( DataSetIException error ) {
      // error.printErrorStack();
      output_log << "\n\t\t\"Error\": \"HDF5 Dataset exception\"\n\t}\n}";
      return EXIT_FAILURE;
    }

  return EXIT_SUCCESS;  // successfully terminated
} 

int main (int argc, char *argv[]) {
  // local variables
    int       i;
    char      * mem_cursor;
    int32_t   block_header;
    string    file_name;
    string    output_destination;
    bool      must_create_name = false;

  // start output for log purposes using json format
    time_t now;
    time(&now);
    char start_time[sizeof "1970-01-01T00:00:00Z"];
    strftime(start_time, sizeof start_time, "%Y-%m-%dT%H:%M:%SZ", gmtime(&now));
    output_log << "\t\"Decode\": {\n\t\t\"UTC_Time_ISO8601\": \"" << start_time << "\",";

  // Check if machine uses big endian. If so, stop the program since this version is not equipped to handle endianess conversion and files are recorded in little endian
    if (Is_Big_Endian()) {
      output_log << "\n\t\t\"Error\": \"Machine is big_endian. Not compatible with this application\"\n\t}\n}";

      // end program execution
      cout << output_log.str();
      return EXIT_FAILURE;
    } 

  // Check if the machine uses 8 bit as one char (one byte).
    if ( CHAR_BIT != 8) {
      output_log << "\n\t\t\"Error\": \"Machine don`t use 8 bits to represent one char and so is not compatible with this application\"\n\t}\n}";

      // end program execution
      cout << output_log.str();
      return EXIT_FAILURE;
    }

  // Check if arguments were passed
    if (argc < 1) {
      output_log << "\n\t\t\"Error\": \"This program requires at least one argument, the name of the CRFS BIN file to be processed\"\n\t}\n}";

      // end program execution
      cout << output_log.str();
      return EXIT_FAILURE;
    }
  
  // Sweep all arguments passed to the main function
  // TODO: Check if all requiered parameters are set and act accordingly
    for(i=1;i<argc;i++) {
      //
      if (argv[i][0] == PARAMETER_SWITCH){
        switch (argv[i][1]) { 
          case HELP_OPTION :
            output_log << "\n\t\t\"Message\": \"This application converts CRFS Bin files into HDF5 files.\nIt overwrites existing files.\n - Use -f option followed by the CRFS BIN filename to be processed.\n - Use -d option followed by the select detection mode (0 or 1).\n   See documentation for more information\"\n\t}\n}";
            // end program execution
            cout << output_log.str();
            return EXIT_SUCCESS;
          break;
          case DETECTION_MODE_OPTION :
            i++;
            switch (stoi(argv[i])) {
            // no detection simple store the data for later processing
              case NO_DETECTION_OPTION : {
                detection_mode = NO_DETECTION_OPTION;
                break;
              }
              case LEVEL_DETECTION_OPTION : {
                detection_mode = LEVEL_DETECTION_OPTION;
                break;
              }
              default : {
                output_log << "\n\t\t\"Warning\": \"No valid detection mode selected, set to default option 0 (no detection)\",";
                detection_mode = NO_DETECTION_OPTION;
              }
            }
          break;
          case OUTPUT_UNIT_OPTION :
            i++;
            switch (stoi(argv[i])) {
            // no detection simple store the data for later processing
              case DBM : {
                output_unit = DBM;
                output_unit_label = "dBm";
                break;
              }
              case DBUV_M : {
                output_unit = DBUV_M;
                output_unit_label = "dBuV/m";
                break;
              }
              default : {
                output_log << "\n\t\t\"Warning\": \"No valid output measurement unit selected, set to default option 0 (dBm)\",";
                output_unit = DBM;
                output_unit_label = "dBm";
              }
            }
          break;

          case FILENAME_OPTION :
            i++;
            file_name.assign(argv[i]);
          break;

          case OUTPUT_DESTINATION_OPTION :
            i++;
            output_destination.assign(argv[i]);

            // Check if destination is folder or file and act accordinly
              struct stat file_status;
            // If destination exist
              if( stat(output_destination.c_str(),&file_status) == 0 ) {

                // if destination is a directory
                  if( file_status.st_mode & S_IFDIR ) {
                    // set flag to create a name based on the data
                      must_create_name = true;

                    // include the slash in case the path does not include it
                      string last_character = output_destination.substr(output_destination.size()-1,output_destination.size());
                      if ( last_character != "/" ) {
                        output_destination = output_destination.append("/");
                      }
                  }

                // if destination is a file
                  else if( file_status.st_mode & S_IFREG ) {
                      must_create_name = false;
                      output_log << "\n\t\t\"Warning\": \"Output destination  \\\"" << output_destination << "\\\" already exists and will be overritten\",";
                      //  TODO: test if it is possible to overwrite the file to avoid error on the HDF5 library
                  }

                // if exists but is neather a file nor a folder
                  else {  
                      output_log << "\n\t\t\"Error\": \"Output destination  \\\"" << output_destination << "\\\" is neither a file or a folder\"\n\t}\n}";
                      cout << output_log.str();
                      return EXIT_FAILURE;
                  }
              }
            // If destination does not exist
              else {
                // get the path side of the destination
                  size_t path_end_position = output_destination.find_last_of("/\\")+1;
                  string output_path = output_destination.substr(0,path_end_position-1);
                // if path does not exist output error
                  int erro_code = stat(output_path.c_str(),&file_status);
                  if( erro_code != 0 ) {
                    output_log << "\n\t\t\"Error\": \"Output destination  \\\"" << output_path << "\\\" not accessible\"\n\t}\n}";
                    cout << output_log.str();
                    return EXIT_FAILURE;
                  }
                // else just use the name provided for the new file
              }
          break;

          default :
            output_log << "\n\t\t\"Error\": \"Option \\\"" << argv[i] << "\\\" was not recognized. See documentation or use -h option for a quick help\"\n\t}\n}";
            // end program execution
            cout << output_log.str();
            return EXIT_FAILURE;
        }
      } else {
        // print out an warning and keep execution
        output_log << "\n\t\t\"Warning\": \"Call argument \\\"" << argv[i] << "\\\" was not recognized. All options must be preceeded by \\\"-\\\" character. See documentation or use -h option for a quick help\"\n\t}\n}";
      }
    }

// TODO Later versions could change this to use memory mapped file such as boost interprocess and reduce the memory requirement.
    ifstream file (file_name, ios::in|ios::binary|ios::ate);
    if (!file.is_open())
    {
      output_log << "\n\t\t\"Error\": \"Unable to open file: " << file_name << "\"\n\t}\n}";
      cout << output_log.str();
      return EXIT_FAILURE;
    }

  // get the file size
    data_size = file.tellg();

  // allocate memory to store file
    memblock = new char [data_size];

  // read file to memory and close it
    file.seekg (0, ios::beg);
    file.read (memblock, data_size);
    file.close();

  // Set cursor to the memory point where file is to be processed
    mem_cursor = memblock; 
  
  //output reference to the file read and selected detection mode
    output_log << "\n\t\t\"Input File Name\": \"" << file_name << "\",";

  // read first 4 bytes as number and check if the file version is compatible
    block_header = *(reinterpret_cast<int32_t *>(mem_cursor));
    if (block_header!=22){
      Finish(false,"File isn`t a CRFS DATA FILE V022, not compatible with this application");
      cout << output_log.str();
      return EXIT_FAILURE;
    }

  // Skip to the human readable part of the header (32 bytes) and jump to the beginning of the first block
    mem_cursor = mem_cursor + 36;
  
  // While there are blocks to read, keep reading
    do {
      mem_cursor = Block_Read(mem_cursor);

    } while ( mem_cursor < memblock+data_size  );

  // if a detection mode was selected, perform channel processing
    if ( detection_mode != NO_DETECTION_OPTION ) {
      // perform channel processing
      channel_processing();
    }

  // compose the name for the output file
    if ( must_create_name ) {
      // create standardized filename based on the data site information and timestamp
      stringstream HDF5_filename;
      HDF5_filename <<  site.start_clock.tv_sec << "_" << site.latitude.mean_value << "_" << site.longitude.mean_value << ".h5";
      output_destination = output_destination + HDF5_filename.str();
    }

  // Store data on the HDF5 file
  string start_time_string(start_time);
  if (store_hdf5(start_time_string,output_destination)==EXIT_FAILURE) {
      output_log << "\n\t\t\"Error\": \"Process can not write HDF5 file\"\n\t}\n}";
      // end program execution
      cout << output_log.str();
      return EXIT_FAILURE;
  }

  // if successfull processing finish processing by releasing memory and moving file
    Finish(true,"Success processing file:");

    cout << output_log.str();
    return EXIT_SUCCESS;
}