Merge pull request #64 from HYPERNETS/writer_fill

fill nans on write

Author: shunt16

hypernets_processor/data_io/hypernets_writer.py

@@ -4,8 +4,10 @@
 
 from hypernets_processor.version import __version__
 import os
+import numpy as np
 from datetime import datetime
 
+
 """___Authorship___"""
 __author__ = "Sam Hunt"
 __created__ = "12/2/2020"
@@ -117,6 +119,8 @@ def _write_netcdf(ds, path, compression_level=None):
             var_encoding.update(ds[var_name].encoding)
             encoding.update({var_name: var_encoding})
 
+        ds = HypernetsWriter.fill_ds(ds)
+
         ds.to_netcdf(path, format="netCDF4", engine="netcdf4", encoding=encoding)
 
     @staticmethod
@@ -140,6 +144,24 @@ def _write_csv(ds, path):
             for meta_name in ds.attrs.keys():
                 f.write(meta_name + ": " + ds.attrs[meta_name] + "\n")
 
+    @staticmethod
+    def fill_ds(ds):
+        """
+        Fill nan's in ds will fillValue
+
+        :type ds: xarray.Dataset
+        :param ds: dataset
+
+        :return: filled data
+        :rtype: xarray.Dataset
+        """
+
+        for variable in ds.variables.keys():
+            idx = np.where(np.isnan(ds[variable]))
+            ds[variable][idx] = ds[variable]._FillValue
+
+        return ds
+
 
 if __name__ == "__main__":
     pass

hypernets_processor/data_io/tests/test_hypernets_writer.py

@@ -4,10 +4,12 @@
 
 import unittest
 from unittest.mock import patch, MagicMock
+from hypernets_processor.data_io.dataset_util import DatasetUtil
 from hypernets_processor.data_io.hypernets_writer import HypernetsWriter
 from hypernets_processor.test.test_functions import setup_test_context
 from hypernets_processor.version import __version__
 from xarray import Dataset
+import numpy as np
 
 
 '''___Authorship___'''
@@ -81,6 +83,19 @@ def test__write_netcdf(self):
 
         ds.to_netcdf.assert_called_once_with(path, encoding={}, engine='netcdf4', format='netCDF4')
 
+    def test_fill_ds(self):
+        ds = Dataset()
+        ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32)
+        ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32)
+
+        ds["array_variable1"][2, 3] = np.nan
+        ds["array_variable2"][2, 3] = np.nan
+
+        HypernetsWriter.fill_ds(ds)
+
+        self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36))
+        self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
+
 
 if __name__ == '__main__':
     unittest.main()