Chardata plus encoded datasets prerebase

lib/iris/fileformats/_nc_load_rules/helpers.py

@@ -708,13 +708,13 @@ def build_and_add_global_attributes(engine: Engine):
             ),
         )
         if problem is not None:
-            stack_notes = problem.stack_trace.__notes__
+            stack_notes = problem.stack_trace.__notes__  # type: ignore[attr-defined]
             if stack_notes is None:
                 stack_notes = []
             stack_notes.append(
                 f"Skipping disallowed global attribute '{attr_name}' (see above error)"
             )
-            problem.stack_trace.__notes__ = stack_notes
+            problem.stack_trace.__notes__ = stack_notes  # type: ignore[attr-defined]
 
 
 ################################################################################
@@ -1536,14 +1536,14 @@ def build_and_add_dimension_coordinate(
     )
     if problem is not None:
         coord_var_name = str(cf_coord_var.cf_name)
-        stack_notes = problem.stack_trace.__notes__
+        stack_notes = problem.stack_trace.__notes__  # type: ignore[attr-defined]
         if stack_notes is None:
             stack_notes = []
         stack_notes.append(
             f"Failed to create {coord_var_name} dimension coordinate:\n"
             f"Gracefully creating {coord_var_name!r} auxiliary coordinate instead."
         )
-        problem.stack_trace.__notes__ = stack_notes
+        problem.stack_trace.__notes__ = stack_notes  # type: ignore[attr-defined]
         problem.handled = True
 
         _ = _add_or_capture(
@@ -1643,9 +1643,13 @@ def _add_auxiliary_coordinate(
 
     # Determine the name of the dimension/s shared between the CF-netCDF data variable
     # and the coordinate being built.
-    common_dims = [
-        dim for dim in cf_coord_var.dimensions if dim in engine.cf_var.dimensions
-    ]
+    coord_dims = cf_coord_var.dimensions
+    # if cf._is_str_dtype(cf_coord_var):
+    #     coord_dims = coord_dims[:-1]
+    datavar_dims = engine.cf_var.dimensions
+    # if cf._is_str_dtype(engine.cf_var):
+    #     datavar_dims = datavar_dims[:-1]
+    common_dims = [dim for dim in coord_dims if dim in datavar_dims]
     data_dims = None
     if common_dims:
         # Calculate the offset of each common dimension.

lib/iris/fileformats/cf.py

@@ -26,7 +26,7 @@
 
 import iris.exceptions
 import iris.fileformats._nc_load_rules.helpers as hh
-from iris.fileformats.netcdf import _thread_safe_nc
+from iris.fileformats.netcdf import _bytecoding_datasets, _thread_safe_nc
 from iris.mesh.components import Connectivity
 import iris.util
 import iris.warnings
@@ -67,7 +67,9 @@
 
 # NetCDF returns a different type for strings depending on Python version.
 def _is_str_dtype(var):
-    return np.issubdtype(var.dtype, np.bytes_)
+    # N.B. use 'datatype' not 'dtype', to "look inside" variable wrappers which
+    #  represent 'S1' type data as 'U<xx>'.
+    return isinstance(var.datatype, np.dtype) and np.issubdtype(var.datatype, np.bytes_)
 
 
 ################################################################################
@@ -788,50 +790,63 @@ def cf_label_data(self, cf_data_var):
                 % type(cf_data_var)
             )
 
-        # Determine the name of the label string (or length) dimension by
-        # finding the dimension name that doesn't exist within the data dimensions.
-        str_dim_name = list(set(self.dimensions) - set(cf_data_var.dimensions))
-
-        if len(str_dim_name) != 1:
-            raise ValueError(
-                "Invalid string dimensions for CF-netCDF label variable %r"
-                % self.cf_name
-            )
-
-        str_dim_name = str_dim_name[0]
-        label_data = self[:]
-
-        if ma.isMaskedArray(label_data):
-            label_data = label_data.filled()
-
-        # Determine whether we have a string-valued scalar label
-        # i.e. a character variable that only has one dimension (the length of the string).
-        if self.ndim == 1:
-            label_string = b"".join(label_data).strip()
-            label_string = label_string.decode("utf8")
-            data = np.array([label_string])
-        else:
-            # Determine the index of the string dimension.
-            str_dim = self.dimensions.index(str_dim_name)
-
-            # Calculate new label data shape (without string dimension) and create payload array.
-            new_shape = tuple(
-                dim_len for i, dim_len in enumerate(self.shape) if i != str_dim
-            )
-            string_basetype = "|U%d"
-            string_dtype = string_basetype % self.shape[str_dim]
-            data = np.empty(new_shape, dtype=string_dtype)
-
-            for index in np.ndindex(new_shape):
-                # Create the slice for the label data.
-                if str_dim == 0:
-                    label_index = (slice(None, None),) + index
-                else:
-                    label_index = index + (slice(None, None),)
-
-                label_string = b"".join(label_data[label_index]).strip()
-                label_string = label_string.decode("utf8")
-                data[index] = label_string
+        # # Determine the name of the label string (or length) dimension by
+        # # finding the dimension name that doesn't exist within the data dimensions.
+        # str_dim_names = list(set(self.dimensions) - set(cf_data_var.dimensions))
+        # n_nondata_dims = len(str_dim_names)
+        #
+        # if n_nondata_dims == 0:
+        #     # *All* dims are shared with the data-variable.
+        #     # This is only ok if the data-var is *also* a string type.
+        #     dim_ok = _is_str_dtype(cf_data_var)
+        #     # In this case, we must just *assume* that the last dimension is "the"
+        #     #  string dimension
+        #     str_dim_name = self.dimensions[-1]
+        # else:
+        #     # If there is exactly one non-data dim, that is the one we want
+        #     dim_ok = len(str_dim_names) == 1
+        #     (str_dim_name,) = str_dim_names
+        #
+        # if not dim_ok:
+        #     raise ValueError(
+        #         "Invalid string dimensions for CF-netCDF label variable %r"
+        #         % self.cf_name
+        #     )
+
+        data = self[:]
+        # label_data = self[:]
+        #
+        # if ma.isMaskedArray(label_data):
+        #     label_data = label_data.filled(b"\0")
+        #
+        # # Determine whether we have a string-valued scalar label
+        # # i.e. a character variable that only has one dimension (the length of the string).
+        # if self.ndim == 1:
+        #     label_string = b"".join(label_data).strip()
+        #     label_string = label_string.decode("utf8")
+        #     data = np.array([label_string])
+        # else:
+        #     # Determine the index of the string dimension.
+        #     str_dim = self.dimensions.index(str_dim_name)
+        #
+        #     # Calculate new label data shape (without string dimension) and create payload array.
+        #     new_shape = tuple(
+        #         dim_len for i, dim_len in enumerate(self.shape) if i != str_dim
+        #     )
+        #     string_basetype = "|U%d"
+        #     string_dtype = string_basetype % self.shape[str_dim]
+        #     data = np.empty(new_shape, dtype=string_dtype)
+        #
+        #     for index in np.ndindex(new_shape):
+        #         # Create the slice for the label data.
+        #         if str_dim == 0:
+        #             label_index = (slice(None, None),) + index
+        #         else:
+        #             label_index = index + (slice(None, None),)
+        #
+        #         label_string = b"".join(label_data[label_index]).strip()
+        #         label_string = label_string.decode("utf8")
+        #         data[index] = label_string
 
         return data
 
@@ -1361,7 +1376,11 @@ def __init__(self, file_source, warn=False, monotonic=False):
         if isinstance(file_source, str):
             # Create from filepath : open it + own it (=close when we die).
             self._filename = os.path.expanduser(file_source)
-            self._dataset = _thread_safe_nc.DatasetWrapper(self._filename, mode="r")
+            if _bytecoding_datasets.DECODE_TO_STRINGS_ON_READ:
+                ds_type = _bytecoding_datasets.EncodedDataset
+            else:
+                ds_type = _thread_safe_nc.DatasetWrapper
+            self._dataset = ds_type(self._filename, mode="r")
             self._own_file = True
         else:
             # We have been passed an open dataset.