h5dumpImport.m

(* Mathematica Package *)
   
(* :Title: h5dumpImport *)

(* :Author: Michael Morris *)

(* :Summary: This package imports data from a HDF5 dataset with compound data that has been dumped by h5dump. *)

(* :Context: h5dumpImport` *)

(* :Package Version: 1.0 *)

(* :Copyright: Copyright (c) 2012 Michael Morris -- This software is released under MIT Open Source license *)

(* :History: Version 1.0, April 2012. *)

(* :Keywords: import, HDF5, dataset, compound, h5dump *)

(* :Mathematica Version: 8.0 *)

(* ==========================================================================
   ==========================================================================
   Note: Some comments include sections of the ASCII dump of an HDF5 dataset.
   Those sections were taken from the file name:
   h5dump_testData.h5_AllDatatypes.txt                                       
   ==========================================================================
   ========================================================================== *)

BeginPackage["h5dumpImport`"]

Unprotect @@ Names["h5dumpImport`*"];
ClearAll @@ Names["h5dumpImport`*"];

h5dumpImport::usage = "h5dumpImport
The object definition used to import an ASCII dump of an HDF5 dataset generated by the h5dump command line tool.";

h5dump::usage = "h5dump[h5dumpLocation, HDF5PatheAndFilename, datasetName, dumpDirectory]
Convenience function to generate an ASCII dump of an HDF5 dataset using the h5dump command line tool.
h5dumpLocation - (String) The location to the h5dump command line tool.
HDF5PatheAndFilename - (String) The path and filename to HDF5 (.h5) file.
datasetName - (String) The dataset in the specified HDF5 (.h5) file.
dumpDirectory - (String - optional) The directory to write the dump file to.  If no value is specified, then $TemporaryDirectory is used.
returns - (String) If successful the path and filename to the ASCII dump of an HDF5 dataset, Null otherwise";

h5dumpImportNew::usage = "h5dumpImportNew[h5dumpImport[], h5dumpFilePathAndFilename]
Create a new h5dumpImport.
h5dumpFilePathAndFilename - (String) The path and filename of the ASCII dump of an HDF5 dataset.
returns - (h5dumpImport) If the header of the ASCII dump of an HDF5 dataset is successfully read, the h5dumpImport, otherwise Null.";

h5dumpImportReset::usage = "h5dumpImportReset[]
Resets the h5dumpImport to a state before any data has been read.
returns - (Symbol) True if the header of the ASCII dump of an HDF5 dataset is successfully read, otherwise False.";

h5dumpImportClose::usage = "h5dumpImportClose[]
Close the stream used to read the ASCII dump of an HDF5 dataset.
returns - (Null)";

h5dumpImportData::usage = "h5dumpImportData[amount]
Retrieves an amount of data from  the header of the ASCII dump of an HDF5 dataset.
amount - (Integer|All - optional) The amount of data to retrieve. If no value is specified, only one piece of data will be retrieved. If an Integer value is specified, that amount will be retrieved, if remaining.  If All is specified, all of the remaining data will be retrieved. 
returns - (List of Lists | List) If no value is specified, a List of data is returned.  If a value (Integer value or All) is specified, a List of Lists of data is returned.";

h5dumpImportFastForward::usage = "h5dumpImportFastForward[hdf5Index]
Fast forwards to just before the specified hdf5Index, so that the next call to h5dumpImportData reads the data for the specified hdf5Index.
hdf5Index - (Integer) The HDF5 index of the data to be read next. Note that the HDF5 index is zero based while the position number is 1 based.
returns - (Integer) The hdf5Index of the position after the function has completed. Note that the HDF5 index is zero based while the position number is 1 based.";

h5dumpImportHdf5Name::usage = "h5dumpImportHdf5Name[]
Retrieves the name of the HDF5 (.h5) file used to generate the ASCII dump of an HDF5 dataset.
returns - (String) The name of the HDF5 (.h5) file used to generate the ASCII dump of an HDF5 dataset.";

h5dumpImportDatasetName::usage = "h5dumpImportDatasetName[]
Retrieves the name of the HDF5 (.h5) dataset used to generate the ASCII dump of an HDF5 dataset.
returns - (String) The name of the HDF5 (.h5) dataset used to generate the ASCII dump of an HDF5 dataset.";

h5dumpImportDataLength::usage = "h5dumpImportDataLength[]
Retrieves the length of the data of the ASCII dump of an HDF5 dataset.
returns - (Integer) The length of the data of the ASCII dump of an HDF5 dataset.";

h5dumpImportDataTypes::usage = "h5dumpImportDataTypes[]
Retrieves the datatypes for each element of the data in the HDF5 (.h5) dataset.
returns - (List of Symbols) List of the datatypes for each element of the data in the HDF5 (.h5) dataset.";

h5dumpImportDataNames::usage = "h5dumpImportDataNames[]
Retrieves the names for each element of the data in the HDF5 (.h5) dataset.
returns - (List of Strings) List of the names for each element of the data in the HDF5 (.h5) dataset.";

Begin["`Private`"]

(* Data Structure ----------------------------------------------------------- *) 

(* String that contains the path and filename of the ASCII dump of an HDF5 dataset. *)
h5dumpFilePathAndFilename[_];

(* The stream used to read the the ASCII dump of an HDF5 dataset. *)
stream[_] :=
    Null;

(* The name of the HDF5 (.h5) file used to generate the ASCII dump of an HDF5 dataset. *)
hdf5Name[_];

(* The name of the HDF5 (.h5) dataset used to generate the ASCII dump of an HDF5 dataset. *)
datasetName[_];

(* The length of the data of the ASCII dump of an HDF5 dataset. *)
dataLength[_];

(* The datatypes for each element of the data in the HDF5 (.h5) dataset. *)
dataTypes[_];

(* The names for each element of the data in the HDF5 (.h5) dataset. *)
dataNames[_];

(* The 1 based position of the last data read from the h5 dump file. Also the amount of data that has been read in. *)
dataPosition[_];

(* Functions ---------------------------------------------------------------- *)

h5dump[h5dumpLocation_String, HDF5PatheAndFilename_String, datasetName_String, dumpDirectory_String:$TemporaryDirectory] :=
    Module[ {dumpPathAndFilename, command, result},
        If[ !FileExistsQ[h5dumpLocation],
            Message[h5dump::h5dumpDoesNotExist, h5dumpLocation];
            Return[Null];
        ];
        If[ !FileExistsQ[HDF5PatheAndFilename],
            Message[h5dump::HDF5PathAndFilenameDoesNotExist, HDF5PatheAndFilename];
            Return[Null];
        ];
        If[ !DirectoryQ[dumpDirectory],
            Message[h5dump::DumpDirectoryDoesNotExist, dumpDirectory];
            Return[Null];
        ];
        dumpPathAndFilename = FileNameJoin[{dumpDirectory,
            "h5dump" <> "_" <> 
            FileBaseName[HDF5PatheAndFilename] <> "." <> FileExtension[HDF5PatheAndFilename] <>
            StringReplace[datasetName, "/" -> "_"] <>
            ".txt"}];
        If[ FileExistsQ[dumpPathAndFilename],
            Return[dumpPathAndFilename];
        ];
        command = h5dumpLocation <> " -d " <> datasetName <> " " <> HDF5PatheAndFilename <> " > " <> dumpPathAndFilename;
        If[ 0 != (result = Run[command]),
            Message[h5dump::ErrorExecutingCommand, command, result];
            Return[Null];
        ];
        
        (* Return dumpPathAndFilename. *)
        dumpPathAndFilename
    ];
h5dump::h5dumpDoesNotExist = "h5dump does not exist at [`1`]";
h5dump::HDF5PathAndFilenameDoesNotExist = "The HDF5 file does not exist at [`1`]";
h5dump::DumpDirectoryDoesNotExist = "The dump directory does not exist at [`1`]";
h5dump::ErrorExecutingCommand = "Error ['2'] received while executing command [`1`]";

h5dumpImport/:h5dumpImportNew[h5dumpImport[], h5dumpFilePathAndFilenameVal_String] :=
Module[ {tag, newh5dumpImport},
    If[ !FileExistsQ[h5dumpFilePathAndFilenameVal],
        Message[h5dumpImportNew::PathAndFilenameDoesNotExist, h5dumpFilePathAndFilenameVal];
        Return[Null];
    ];
    tag = Unique[];
    newh5dumpImport = h5dumpImport[tag];
    h5dumpFilePathAndFilename[tag] = h5dumpFilePathAndFilenameVal;
    If[ h5dumpImport[tag].h5dumpImportReset[],
        (* Reset success, return the h5dumpImport. *)
        newh5dumpImport(*,*)
    ,
        (* Reset failure. *)
        h5dumpImport[tag].h5dumpImportClose[];
        (* Return Null. *)
        Null
    ]
]
h5dumpImportNew::PathAndFilenameDoesNotExist = "The h5dump file specified by [`1`] does not exist";

h5dumpImport/:h5dumpImport[tag_].h5dumpImportReset[] :=
(
    (* Open the stream. *)
    If[ !Null === stream[tag],
        h5dumpImport[tag].h5dumpImportClose[];
    ];
    stream[tag]  = OpenRead[h5dumpFilePathAndFilename[tag]];
    hdf5Name[tag] = "";
    datasetName[tag] = "";
    dataLength[tag] = 0;
    dataTypes[tag] = {};
    dataNames[tag] = {};
    dataPosition[tag] = 0;
    
    (* Return the status of attempting to read up to the first data item. *)
    (h5dumpImport[tag].ReadHDF5[] &&
    h5dumpImport[tag].ReadDATASET[] &&
    h5dumpImport[tag].ReadDATATYPE[] &&
    h5dumpImport[tag].ReadDATASPACE[] &&
    h5dumpImport[tag].ReadDATA[]) 
);
h5dumpImportReset::UnexpectedValue = "While attempting to read value[`1`], read[`2`]"; 


h5dumpImport/:h5dumpImport[tag_].h5dumpImportClose[] :=
(
    If[ !Null === stream[tag],
        Close[stream[tag]];
        stream[tag] = Null;
    ];
);

h5dumpImport/:h5dumpImport[tag_].h5dumpImportData[amount:(_Integer|All)] :=
(
    If[ All===amount,
        Array[h5dumpImport[tag].ReadData[] &, dataLength[tag] - dataPosition[tag]](*,*)
    ,
        Array[h5dumpImport[tag].ReadData[] &, Min[amount, dataLength[tag] - dataPosition[tag]]]
    ]
);

h5dumpImport/:h5dumpImport[tag_].h5dumpImportData[] :=
If[ dataPosition[tag] < dataLength[tag],
    h5dumpImport[tag].ReadData[](*,*)
    ,
    Null
]

h5dumpImport/:h5dumpImport[tag_].h5dumpImportFastForward[hdf5Index_Integer] :=
Module[ {amountOfDataToSkip, linesToSkip, i},
    If[ hdf5Index > dataPosition[tag] && hdf5Index < dataLength[tag],
        amountOfDataToSkip = hdf5Index - dataPosition[tag];
        linesToSkip = amountOfDataToSkip * (2 + Length[dataTypes[tag]]);
        For[i = 0, i<linesToSkip, i++, 
          Read[stream[tag], String]
        ];
        dataPosition[tag] += amountOfDataToSkip;
    ];
    dataPosition[tag] - 1
];

h5dumpImport/:h5dumpImport[tag_].h5dumpImportHdf5Name[] := hdf5Name[tag];

h5dumpImport/:h5dumpImport[tag_].h5dumpImportDatasetName[] := datasetName[tag];

h5dumpImport/:h5dumpImport[tag_].h5dumpImportDataLength[] := dataLength[tag];

h5dumpImport/:h5dumpImport[tag_].h5dumpImportDataTypes[] := dataTypes[tag];

h5dumpImport/:h5dumpImport[tag_].h5dumpImportDataNames[] := dataNames[tag];

h5dumpImport/:h5dumpImport[tag_].ReadHDF5[] :=
Module[ {value},
    
    (* [HDF5 "/tmp/testData.h5" {] *)
    
    (*  Read [HDF5] *)
    If[ "HDF5" != (value = Read[stream[tag], Word]),
        Message[h5dumpImportReset::UnexpectedValue, "HDF5", value];
        Return[False];
    ];
    
    (* Read ["/tmp/testData.h5"] *)
    hdf5Name[tag] = h5dumpImport[tag].ReadQuotedString[];
    
    (* Read [ {] *)
    Read[stream[tag], String];

    (* Return True *)
    True
];

h5dumpImport/:h5dumpImport[tag_].ReadDATASET[] :=
Module[ {value},
    (* [DATASET "/AllDatatypes" {] *)
    
    (* Read [DATASET] *)
    If[ "DATASET" != (value = Read[stream[tag], Word]),
        Message[h5dumpImportReset::UnexpectedValue, "DATASET", value];
        Return[False];
    ];
    
    (* Read [ "/AllDatatypes"] *)
    datasetName[tag] = h5dumpImport[tag].ReadQuotedString[];
    
    (* Read [ {] *)
    Read[stream[tag], String];

    (* Return True *)
    True
];

h5dumpImport/:h5dumpImport[tag_].ReadDATATYPE[] :=
Module[ {value},
    (* [   DATATYPE  H5T_COMPOUND {
      H5T_STD_I8LE "AsByte";
      H5T_STD_U8LE "AsUnsignedByte";
      H5T_STD_I16LE "AsShort";
      H5T_STD_U16LE "AsUnsignedShort";
      H5T_STD_I32LE "AsInt";
      H5T_STD_U32LE "AsUnsignedInt";
      H5T_STD_I64LE "AsLong";
      H5T_IEEE_F32LE "AsFloat";
      H5T_IEEE_F64LE "AsDouble";
      H5T_STRING {
         STRSIZE 10;
         STRPAD H5T_STR_NULLTERM;
         CSET H5T_CSET_ASCII;
         CTYPE H5T_C_S1;
      } "AsString";
   }] *)
    
    (* Read [   DATATYPE] *)
    If[ "DATATYPE" != (value = Read[stream[tag], Word]),
        Message[h5dumpImportReset::UnexpectedValue, "DATATYPE", value];
        Return[False];
    ];

    (* Read [  H5T_COMPOUND {
      H5T_STD_I8LE "AsByte";
      H5T_STD_U8LE "AsUnsignedByte";
      H5T_STD_I16LE "AsShort";
      H5T_STD_U16LE "AsUnsignedShort";
      H5T_STD_I32LE "AsInt";
      H5T_STD_U32LE "AsUnsignedInt";
      H5T_STD_I64LE "AsLong";
      H5T_IEEE_F32LE "AsFloat";
      H5T_IEEE_F64LE "AsDouble";
      H5T_STRING {
         STRSIZE 10;
         STRPAD H5T_STR_NULLTERM;
         CSET H5T_CSET_ASCII;
         CTYPE H5T_C_S1;
      } "AsString";
   }] *)
    h5dumpImport[tag].ReadDataType[];
    
    (* Return True *)
    True
];

h5dumpImport/:h5dumpImport[tag_].ReadDATASPACE[] :=
Module[ {value},
    (* [   DATASPACE  SIMPLE { ( 3 ) / ( H5S_UNLIMITED ) }] *)
    
    (* Read [   DATASPACE] *)
    If[ "DATASPACE" != (value = Read[stream[tag], Word]),
        Message[h5dumpImportReset::UnexpectedValue, "DATASPACE", value];
        Return[False];
    ];

    (* Read [  SIMPLE { ( 3] *)
    dataLength[tag] = Read[stream[tag], {Word, Character, Character, Character, Character, Number}][[6]];

    (* Read [ ) / ( H5S_UNLIMITED ) }] *)
    Read[stream[tag], String];
    
    (* Return True *)
    True
];

h5dumpImport/:h5dumpImport[tag_].ReadDATA[] :=
Module[ {value},
    (* [   DATA {] *)

    (* Read [   DATA] *)
    If[ "DATA" != (value = Read[stream[tag], Word]),
        Message[h5dumpImportReset::UnexpectedValue, "DATA", value];
        Return[False];
    ];
    
    (* Read [ {] *)
    Read[stream[tag], String];
    
    (* Return True *)
    True
];

h5dumpImport/:h5dumpImport[tag_].ReadDataType[] :=
Module[ {dataType},
    (* [H5T_COMPOUND {
      H5T_STD_I8LE "AsByte";
      H5T_STD_U8LE "AsUnsignedByte";
      H5T_STD_I16LE "AsShort";
      H5T_STD_U16LE "AsUnsignedShort";
      H5T_STD_I32LE "AsInt";
      H5T_STD_U32LE "AsUnsignedInt";
      H5T_STD_I64LE "AsLong";
      H5T_IEEE_F32LE "AsFloat";
      H5T_IEEE_F64LE "AsDouble";
      H5T_STRING {
         STRSIZE 10;
         STRPAD H5T_STR_NULLTERM;
         CSET H5T_CSET_ASCII;
         CTYPE H5T_C_S1;
      } "AsString";
   }] *)
    
    (* Read [  H5T_COMPOUND] *)
    (* Read [      H5T_STD_I8LE] *)
    (* Read [      H5T_STD_U8LE] *)
    (* Read [      H5T_STD_I16LE] *)
    (* Read [      H5T_STD_U16LE] *)
    (* Read [      H5T_STD_I32LE] *)
    (* Read [      H5T_STD_U32LE] *)
    (* Read [      H5T_STD_I64LE] *)
    (* Read [      H5T_IEEE_F32LE] *)
    (* Read [      H5T_IEEE_F64LE] *)
    (* Read [      H5T_STRING] *)
    dataType = Read[stream[tag], Word];
    Switch[dataType,
        
    "H5T_COMPOUND", (* ------------------------------------------------------ *)
    (* Read [ {] *)
    Read[stream[tag], String];
    (* Read data types *)
    While[!h5dumpImport[tag].ReadDataType[]](*,*)
    ,
    
    "H5T_IEEE_F32LE", (* ---------------------------------------------------- *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsFloat"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_IEEE_F64LE", (* ---------------------------------------------------- *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsDouble"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_I8LE", (* ------------------------------------------------------ *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsByte"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_I16LE", (* ----------------------------------------------------- *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsShort"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_I32LE", (*------------------------------------------------------ *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsInt"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_I64LE", (*------------------------------------------------------ *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsLong"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_U8LE", (*------------------------------------------------------- *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsUnsignedByte"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_U16LE", (*------------------------------------------------------ *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsUnsignedShort"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
    
    "H5T_STD_U32LE", (*------------------------------------------------------ *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "AsUnsignedInt"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
     
    "H5T_STRING", (*--------------------------------------------------------- *)
    AppendTo[dataTypes[tag], Word];
    (* Read [{
         STRSIZE 10;
         STRPAD H5T_STR_NULLTERM;
         CSET H5T_CSET_ASCII;
         CTYPE H5T_C_S1;
      }] *)
    While["}" != Read[stream[tag], Character]];
    (* Read [ "AsString"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];(*,*)
    ,
     
    "}", (* ----------------------------------------------------------------- *)
    (* Return True - matches { of H5T_COMPOUND *) 
    Return[True];(*,*)
    ,
    
    _, (* ------------------------------------------------------------------- *)
    AppendTo[dataTypes[tag], Number];
    (* Read [ "????"] *)
    AppendTo[dataNames[tag], h5dumpImport[tag].ReadQuotedString[]];
    (* Read [;] *)
    Read[stream[tag], Character];
    ];
    
    (* Return False - does not match { of H5T_COMPOUND. *)
    False
]

h5dumpImport/:h5dumpImport[tag_].ReadData[] :=
Module[ {data},
    dataPosition[tag]++;
    (* [   (0): {
         1,
         11,
         111,
         1111,
         11111,
         111111,
         1111111,
         1.1,
         11.11,
         "one"
      },] *)
      
    (* [   (2): {
         3,
         33,
         333,
         3333,
         33333,
         333333,
         3333333,
         3.3,
         33.33,
         "three"
      }] *)
      
    (* Read [   (0): {] *)
    (* Read [   (2): {] *)
    Read[stream[tag], String];
 
    (* Read [        1,
         11,
         111,
         1111,
         11111,
         111111,
         1111111,
         1.1,
         11.11,
         "one"] *)
    (* Read [         3,
         33,
         333,
         3333,
         33333,
         333333,
         3333333,
         3.3,
         33.33,
         "three"] *)
    data = Map[
        If[ #===Word,
            StringTrim[Read[stream[tag], String], (" " | "," | "\"") ...](*,*)
        ,
            ToExpression[StringTrim[Read[stream[tag], String], (" " | ",") ...]]
        ]
        &, dataTypes[tag]];
        
    (* Read [      },] *)
    (* Read [      }] *)
    Read[stream[tag], String];
    
    (* Return the data. *)
    data
]

h5dumpImport /: h5dumpImport[tag_].ReadQuotedString[] :=
Module[ {string, insideQuotes, value},
    string = "";
    insideQuotes = False;
    While[! EndOfFile === (value = Read[stream[tag], Character]),
        Switch[value,
        " ", (* ------------------------------------------------------------- *)
        If[ insideQuotes,
            string = string <> value;
        ];(*,*)
        ,
        
        "\\", (* ------------------------------------------------------------ *)
        value = Read[stream[tag], Character];
        If[ EndOfFile === value,
            string = string <> "\\";
            Break[];
        ];
        If[ "\"" == value,
            string = string <> "\"";
        ];(*,*)
        ,
        
        "\"", (* ------------------------------------------------------------ *)
        If[ insideQuotes,
            Break[];(*,*)
        ,
            insideQuotes = True;
        ];(*,*)
      ,
      
      _, (* ----------------------------------------------------------------- *)
         string = string <> value;
      ];
    ];
    string
];

End[]

Protect @@ Names["h5dumpImport`*"];

EndPackage[]