Fix HDF5 serialization for datetime and date objects, resolve FilesOpenItem serialization bug, and enhance HDF5 serialization tests

Author: PierreRaybaut

CHANGELOG.md

@@ -200,6 +200,25 @@
 
 🛠️ Bug fixes:
 
+* Fixed HDF5 serialization and deserialization for datetime and date objects:
+  * Previously, datetime and date objects were serialized as numerical values (timestamp for datetime, ordinal for date) but were not properly restored as the original object types upon deserialization.
+  * This caused `datetime.datetime` objects to be restored as `float` values and `datetime.date` objects to be restored as `int` values.
+  * The fix ensures that these temporal objects are now correctly restored as their original types, maintaining data integrity across save/load cycles.
+  * Updated the HDF5Reader to detect and convert numerical values back to datetime/date objects when appropriate.
+  * This affects all DataSet instances containing `DateItem` or `DateTimeItem` objects that are saved to and loaded from HDF5 files.
+
+* Fixed `FilesOpenItem` serialization bug in HDF5 files:
+  * Previously, when serializing file paths in `FilesOpenItem`, the paths were encoded to UTF-8 bytes but not properly decoded during deserialization.
+  * This caused file paths to be incorrectly restored as lists of individual characters instead of complete path strings.
+  * The fix ensures that file paths are properly decoded from bytes to strings during HDF5 deserialization.
+  * This resolves data corruption issues when saving and loading datasets containing multiple file selections.
+
+* Enhanced HDF5 serialization test to prevent regressions:
+  * The automatic unit test for HDF5 serialization (`test_loadsave_hdf5.py`) now properly validates dataset integrity after serialization/deserialization cycles.
+  * Previously, the test could pass even when values were corrupted during the save/load process due to improper initialization.
+  * The test now explicitly sets all items to `None` after creating the target dataset, ensuring that deserialized values truly come from the HDF5 file rather than from default initialization.
+  * This improvement helps catch serialization bugs early and prevents future regressions in HDF5 I/O functionality.
+
 * Fixed font hinting preference in `RotatedLabel` initialization for improved text rendering
 
 * Fixed dataset corruption in `DataSetShowGroupBox.get()` when updating widgets with dependencies:

guidata/dataset/dataitems.py

@@ -906,17 +906,20 @@ def __init__(
     ) -> None:
         if isinstance(default, str):
             default = [default]
-        super().__init__(
+        StringItem.__init__(
+            self,
             label,
-            formats=formats,
             default=default,
-            basedir=basedir,
-            all_files_first=all_files_first,
             regexp=regexp,
             help=help,
             check=check,
             allow_none=allow_none,
         )
+        if isinstance(formats, str):
+            formats = [formats]  # type:ignore
+        self.set_prop("data", formats=formats)
+        self.set_prop("data", basedir=basedir)
+        self.set_prop("data", all_files_first=all_files_first)
         self.set_prop("display", func=self.paths_basename)
 
     @staticmethod
@@ -955,6 +958,8 @@ def serialize(
     ) -> None:
         """Serialize this item"""
         value = self.get_value(instance)
+        if value is not None and not isinstance(value, (tuple, list)):
+            value = [value]
         writer.write_sequence([fname.encode("utf-8") for fname in value])
 
     def get_value_from_reader(

guidata/io/h5fmt.py

@@ -10,6 +10,7 @@
 from __future__ import annotations
 
 import datetime
+import numbers
 import sys
 from collections.abc import Callable, Sequence
 from typing import Any
@@ -474,10 +475,10 @@ def write(self, val: Any, group_name: str | None = None) -> None:
             group = self.get_parent_group()
             try:
                 group.attrs[self.option[-1]] = val
-            except TypeError:
+            except TypeError as exc:
                 raise NotImplementedError(
                     "cannot serialize %r of type %r" % (val, type(val))
-                )
+                ) from exc
 
         if group_name:
             self.end(group_name)
@@ -583,6 +584,32 @@ class NoDefault:
     pass
 
 
+def infer_datetime(value: Any) -> Any:
+    """Infer if a numeric value represents a datetime object.
+
+    Args:
+        value: The value to check.
+
+    Returns:
+        The inferred datetime object if applicable, otherwise the original value.
+    """
+    if isinstance(value, (numbers.Real, np.generic)):
+        # Convert NumPy scalars to native Python types for consistency
+        value = value.item() if isinstance(value, np.generic) else value
+
+        # datetime.datetime: seconds since epoch
+        if isinstance(value, float) and 1e8 < value < 2e9:
+            return datetime.datetime.fromtimestamp(value)
+
+        # datetime.date: ordinal days (typical range)
+        if isinstance(value, numbers.Integral) and 50000 < value < 1000000:
+            try:
+                return datetime.date.fromordinal(value)
+            except ValueError:
+                pass
+    return value
+
+
 class HDF5Reader(HDF5Handler):
     """
     Reader for HDF5 files. Inherits from HDF5Handler.
@@ -641,7 +668,18 @@ def read(
             self.end(group_name)
         return val
 
-    def read_any(self) -> str | bytes:
+    def read_any(
+        self,
+    ) -> (
+        str
+        | bytes
+        | int
+        | float
+        | datetime.date
+        | datetime.datetime
+        | list[Any]
+        | np.ndarray
+    ):
         """
         Read a value from the current group as a generic type.
 
@@ -659,8 +697,7 @@ def read_any(self) -> str | bytes:
             value = self.read_sequence()
         if isinstance(value, bytes):
             return value.decode("utf-8")
-        else:
-            return value
+        return infer_datetime(value)
 
     def read_bool(self) -> bool | None:
         """
@@ -756,7 +793,7 @@ def read_dict(self) -> dict[str, Any]:
         for key, value in dict_group.attrs.items():
             if key == DICT_NAME:
                 continue
-            dict_val[key] = value
+            dict_val[key] = infer_datetime(value)
         for key in dict_group:
             with self.group(key):
                 try:

guidata/tests/dataset/test_all_features.py

@@ -34,7 +34,7 @@
 class SubDataSet(gds.DataSet):
     dir = gds.DirectoryItem("Directory", TEMPDIR)
     fname = gds.FileOpenItem("Single file (open)", ("csv", "eta"), FILE_CSV.name)
-    fnames = gds.FilesOpenItem("Multiple files", "csv", FILE_CSV.name)
+    fnames = gds.FilesOpenItem("Multiple files", "csv", [FILE_CSV.name])
     fname_s = gds.FileSaveItem("Single file (save)", "eta", FILE_ETA.name)
 
 

guidata/tests/dataset/test_all_items.py

@@ -82,15 +82,15 @@ class Parameters(gds.DataSet):
     as well as special characters (α, β, γ, δ, ...)
     """
 
-    dir = gds.DirectoryItem("Directory")
-    preview = gds.TextItem("File names preview")
+    dir = gds.DirectoryItem("Directory", default=TEMPDIR)
+    preview = gds.TextItem("File names preview", default="-")
     option = gds.ChoiceItem("Option", (("1", "first choice"), ("2", "second choice")))
 
     fname = gds.FileOpenItem("Open file", ("csv", "eta"), FILE_CSV.name)
-    fnames = gds.FilesOpenItem("Open files", "csv", FILE_CSV.name)
+    fnames = gds.FilesOpenItem("Open files", "csv", [FILE_CSV.name])
     fname_s = gds.FileSaveItem("Save file", "eta", FILE_ETA.name)
-    string = gds.StringItem("String")
-    text = gds.TextItem("Text")
+    string = gds.StringItem("String", default="default string !?")
+    text = gds.TextItem("Text", default="default\nmultiline\ntext")
     float_slider = gds.FloatItem(
         "Float (with slider)", default=0.5, min=0, max=1, step=0.01, slider=True
     )

guidata/tests/dataset/test_loadsave_hdf5.py

@@ -13,8 +13,10 @@
 
 # guitest: show
 
-import os
+import os.path as osp
+import tempfile
 
+from guidata.dataset import assert_datasets_equal
 from guidata.env import execenv
 from guidata.io import HDF5Reader, HDF5Writer
 from guidata.qthelpers import qt_app_context
@@ -23,28 +25,29 @@
 
 def test_loadsave_hdf5():
     """Test HDF5 I/O"""
-    fname = "test.h5"
-    with qt_app_context():
-        if os.path.exists(fname):
-            os.unlink(fname)
+    with tempfile.TemporaryDirectory() as temp_dir:
+        fname = osp.join(temp_dir, "test.h5")
+        with qt_app_context():
+            p1 = Parameters()
+            # p1.edit()
 
-        p1 = Parameters()
-        if execenv.unattended or p1.edit():
+            # Save to HDF5 file
             writer = HDF5Writer(fname)
             p1.serialize(writer)
             writer.close()
 
             p2 = Parameters()
+            # Set all items to None for testing purposes:
+            for item in p2._items:
+                item.__set__(p2, None)
+
+            # Load from HDF5 file
             reader = HDF5Reader(fname)
             p2.deserialize(reader)
             reader.close()
-            p2.edit()
-            os.unlink(fname)
 
-        # TODO: Uncomment this part of the test, and make it work!
-        # if execenv.unattended:
-        #     assert_datasets_equal(p1, p2, "Parameters do not match after HDF5 I/O")
-        execenv.print("OK")
+            assert_datasets_equal(p1, p2, "Parameters do not match after HDF5 I/O")
+            execenv.print("OK")
 
 
 if __name__ == "__main__":