From 7b492ff45965ba74abac76d41f028978610b748e Mon Sep 17 00:00:00 2001 From: Felipe Alex Hofmann Date: Wed, 8 Jul 2026 06:51:50 -0700 Subject: [PATCH] add missingness --- rdt/transformers/base.py | 2 +- rdt/transformers/categorical.py | 190 +++++++++++++++--- rdt/transformers/numerical.py | 2 +- .../transformers/test_categorical.py | 182 +++++++++++++++++ tests/unit/transformers/test_categorical.py | 171 ++++++++++++++++ 5 files changed, 512 insertions(+), 35 deletions(-) diff --git a/rdt/transformers/base.py b/rdt/transformers/base.py index 97e6c943d..3922fb065 100644 --- a/rdt/transformers/base.py +++ b/rdt/transformers/base.py @@ -113,7 +113,7 @@ def reset_randomization(self): @property def model_missing_values(self): - """Return whether or not a new column is being used to model missing values.""" + """Whether or not a new column is being used to model missing values.""" warnings.warn( "Future versions of RDT will not support the 'model_missing_values' parameter. " "Please switch to using the 'missing_value_generation' parameter instead.", diff --git a/rdt/transformers/categorical.py b/rdt/transformers/categorical.py index 2fdfec112..262c1af59 100644 --- a/rdt/transformers/categorical.py +++ b/rdt/transformers/categorical.py @@ -18,6 +18,13 @@ LOGGER = logging.getLogger(__name__) +def _validate_missing_value_encoding(missing_value_encoding): + if missing_value_encoding not in {'new_category', None}: + raise TransformerInputError( + "'missing_value_encoding' must be one of the following values: None or 'new_category'." + ) + + class UniformEncoder(BaseTransformer): """Transformer for categorical data. @@ -37,6 +44,10 @@ class UniformEncoder(BaseTransformer): order_by (str or None): String defining how to order the data before applying the labels. Options are 'alphabetical', 'numerical' and ``None``. Defaults to ``None``. + missing_value_encoding (str or None): + How to encode missing values. If ``'new_category'``, missing values are encoded as + their own category. If ``None``, missing values are not encoded and remain missing. + Defaults to ``'new_category'``. """ INPUT_SDTYPE = 'categorical' @@ -45,7 +56,7 @@ class UniformEncoder(BaseTransformer): intervals = None dtype = None - def __init__(self, order_by=None): + def __init__(self, order_by=None, missing_value_encoding='new_category'): super().__init__() if order_by not in [None, 'alphabetical', 'numerical_value']: raise TransformerInputError( @@ -53,7 +64,9 @@ def __init__(self, order_by=None): "'alphabetical'" ) + _validate_missing_value_encoding(missing_value_encoding) self.order_by = order_by + self.missing_value_encoding = missing_value_encoding def _order_categories(self, unique_data): nans = pd.isna(unique_data) @@ -126,7 +139,11 @@ def _fit(self, data): Data to fit the transformer to. """ self.dtype = data.dtypes - data = fill_nan_with_none(data) + if self.missing_value_encoding is None: + data = data[~pd.isna(data)] + else: + data = fill_nan_with_none(data) + labels = pd.unique(data) labels = self._order_categories(labels) freq = data.value_counts(normalize=True, dropna=False) @@ -166,8 +183,12 @@ def _transform(self, data): Returns: pandas.Series """ - data_with_none = fill_nan_with_none(data) - unseen_indexes = ~(data_with_none.isin(self.frequencies)) + if self.missing_value_encoding is None: + unseen_indexes = ~(data.isin(self.frequencies)) & ~pd.isna(data) + else: + data = fill_nan_with_none(data) + unseen_indexes = ~(data.isin(self.frequencies)) + if unseen_indexes.any(): # Keep the 3 first unseen categories unseen_categories = list(data.loc[unseen_indexes].unique()) @@ -181,13 +202,19 @@ def _transform(self, data): ) choices = list(self.frequencies.keys()) - size = unseen_indexes.size - data_with_none[unseen_indexes] = np.random.choice(choices, size=size) + if choices: + size = unseen_indexes.size + data[unseen_indexes] = np.random.choice(choices, size=size) def map_labels(label): return np.random.uniform(self.intervals[label][0], self.intervals[label][1]) - return data_with_none.map(map_labels).astype(float) + known_indexes = data.isin(self.frequencies) + result = pd.Series(np.nan, index=data.index, name=data.name, dtype=float) + if known_indexes.any(): + result.loc[known_indexes] = data.loc[known_indexes].map(map_labels).astype(float) + + return result def _reverse_transform(self, data): """Convert float values back to the original categorical values. @@ -199,7 +226,14 @@ def _reverse_transform(self, data): Returns: pandas.Series """ - check_nan_in_transform(data, self.dtype) + if self.missing_value_encoding == 'new_category': + check_nan_in_transform(data, self.dtype) + + # If nothing was learned, reverse-transform everything to missing. + if self.intervals is not None and len(self.intervals) == 0: + result = pd.Series(np.nan, index=data.index, name=data.name) + return try_convert_to_dtype(result, self.dtype) + data = data.clip(0, 1) bins = [0] labels = [] @@ -234,9 +268,15 @@ class OrderedUniformEncoder(UniformEncoder): order (list): A list of all the unique categories for the data. The order of the list determines the label that each category will get. + missing_value_encoding (str or None): + How to encode missing values. If ``'new_category'``, missing values are encoded as + their own category. If ``None``, missing values are not encoded and remain missing. + Defaults to ``'new_category'``. """ - def __init__(self, order): + def __init__(self, order, missing_value_encoding='new_category'): + _validate_missing_value_encoding(missing_value_encoding) + self.missing_value_encoding = missing_value_encoding self.order = fill_nan_with_none(pd.Series(order)) if not self.order.is_unique: error_msg = ( @@ -245,7 +285,7 @@ def __init__(self, order): ) raise TransformerInputError(error_msg) - super().__init__() + super().__init__(missing_value_encoding=missing_value_encoding) def __repr__(self): """Represent initialization of transformer as text. @@ -256,11 +296,25 @@ def __repr__(self): """ class_name = self.__class__.get_name() custom_args = ['order='] + if self.missing_value_encoding != 'new_category': + custom_args.append(f'missing_value_encoding={repr(self.missing_value_encoding)}') + args_string = ', '.join(custom_args) return f'{class_name}({args_string})' + def _get_order(self): + if self.missing_value_encoding is None: + return self.order[~pd.isna(self.order)] + + return self.order + def _check_unknown_categories(self, data): - missing = list(data[~data.isin(self.order)].unique()) + order = self._get_order() + unknown = ~data.isin(order) + if self.missing_value_encoding is None: + unknown &= ~pd.isna(data) + + missing = list(data[unknown].unique()) if len(missing) > 0: raise TransformerInputError( f"Unknown categories '{missing}'. All possible categories must be defined in the " @@ -278,13 +332,22 @@ def _fit(self, data): Data to fit the transformer to. """ self.dtype = data.dtypes - data = fill_nan_with_none(data) + order = self._get_order() + if self.missing_value_encoding is None: + data = data[~pd.isna(data)] + else: + data = fill_nan_with_none(data) + self._check_unknown_categories(data) - category_not_seen = set(self.order.dropna()) != set(data.dropna()) - nans_not_seen = pd.isna(self.order).any() and not pd.isna(data).any() + category_not_seen = set(order.dropna()) != set(data.dropna()) + nans_not_seen = ( + self.missing_value_encoding == 'new_category' + and pd.isna(order).any() + and not pd.isna(data).any() + ) if category_not_seen or nans_not_seen: - unseen_categories = [x for x in self.order if x not in data.array] + unseen_categories = [x for x in order if x not in data.array] categories_to_print = self._get_message_unseen_categories(unseen_categories) LOGGER.info( "For column '%s', some of the provided category values were not present in the" @@ -296,19 +359,21 @@ def _fit(self, data): freq = data.value_counts(normalize=True, dropna=False) freq = 0.9 * freq for category in unseen_categories: - freq[category] = 0.1 / len(unseen_categories) + freq.loc[category] = 0.1 / len(unseen_categories) else: freq = data.value_counts(normalize=True, dropna=False) nan_value = freq[np.nan] if np.nan in freq.index else None - freq = freq.reindex(self.order, fill_value=nan_value).array + freq = freq.reindex(order, fill_value=nan_value).array - self.frequencies, self.intervals = self._compute_frequencies_intervals(self.order, freq) + self.frequencies, self.intervals = self._compute_frequencies_intervals(order, freq) def _transform(self, data): """Map the category to a continuous value.""" - data = fill_nan_with_none(data) + if self.missing_value_encoding == 'new_category': + data = fill_nan_with_none(data) + self._check_unknown_categories(data) return super()._transform(data) @@ -730,6 +795,10 @@ class LabelEncoder(BaseTransformer): - ``'numerical_value'``: Order the categories by numerical value. - ``'alphabetical'``: Order the categories alphabetically. - ``None``: Use the order that the categories appear in when fitting. + missing_value_encoding (str or None): + How to encode missing values. If ``'new_category'``, missing values are encoded as + their own category. If ``None``, missing values are not encoded and remain missing. + Defaults to ``'new_category'``. """ INPUT_SDTYPE = 'categorical' @@ -738,7 +807,7 @@ class LabelEncoder(BaseTransformer): categories_to_values = None dtype = 'O' - def __init__(self, add_noise=False, order_by=None): + def __init__(self, add_noise=False, order_by=None, missing_value_encoding='new_category'): super().__init__() self.add_noise = add_noise if order_by not in [None, 'alphabetical', 'numerical_value']: @@ -747,9 +816,14 @@ def __init__(self, add_noise=False, order_by=None): "'alphabetical'" ) + _validate_missing_value_encoding(missing_value_encoding) self.order_by = order_by + self.missing_value_encoding = missing_value_encoding def _order_categories(self, unique_data): + if len(unique_data) == 0: + return unique_data + if self.order_by == 'alphabetical': if unique_data.dtype.type not in [np.str_, np.object_]: raise TransformerInputError( @@ -782,7 +856,13 @@ def _fit(self, data): Data to fit the transformer to. """ self.dtype = data.dtype - unique_data = pd.unique(data.infer_objects().fillna(np.nan)) + data = data.infer_objects() + if self.missing_value_encoding is None: + data = data[~pd.isna(data)] + else: + data = data.fillna(np.nan) + + unique_data = pd.unique(data) unique_data = self._order_categories(unique_data) self.values_to_categories = dict(enumerate(unique_data)) self.categories_to_values = { @@ -804,19 +884,31 @@ def _transform(self, data): Returns: pd.Series """ - mapped = data.infer_objects().fillna(np.nan).map(self.categories_to_values) - is_null = mapped.isna() - if is_null.any(): + data = data.infer_objects() + if self.missing_value_encoding is None: + mapped = data.map(self.categories_to_values) + unseen_indexes = mapped.isna() & ~pd.isna(data) + else: + data = data.fillna(np.nan) + mapped = data.map(self.categories_to_values) + unseen_indexes = mapped.isna() + + if unseen_indexes.any(): # Select only the first 5 unseen categories to avoid flooding the console. - unseen_categories = set(data[is_null][:5]) + unseen_categories = set(data[unseen_indexes][:5]) warnings.warn( - f'The data contains {is_null.sum()} new categories that were not ' + f'The data contains {unseen_indexes.sum()} new categories that were not ' f'seen in the original data (examples: {unseen_categories}). Assigning ' 'them random values. If you want to model new categories, ' 'please fit the transformer again with the new data.' ) - mapped[is_null] = np.random.randint(len(self.categories_to_values), size=is_null.sum()) + if unseen_indexes.any() and self.categories_to_values: + mapped[unseen_indexes] = np.random.randint( + len(self.categories_to_values), size=unseen_indexes.sum() + ) + + mapped = pd.to_numeric(mapped) if self.add_noise: mapped = mapped.astype(float) @@ -834,7 +926,14 @@ def _reverse_transform(self, data): Returns: pandas.Series """ - check_nan_in_transform(data, self.dtype) + if self.missing_value_encoding == 'new_category': + check_nan_in_transform(data, self.dtype) + + # If nothing was learned, reverse-transform everything to missing. + if self.values_to_categories is not None and len(self.values_to_categories) == 0: + result = pd.Series(np.nan, index=data.index, name=data.name) + return try_convert_to_dtype(result, self.dtype) + if self.add_noise: data = np.floor(data) @@ -860,9 +959,15 @@ class OrderedLabelEncoder(LabelEncoder): add_noise (bool): Whether to generate uniform noise around the label for each category. Defaults to ``False``. + missing_value_encoding (str or None): + How to encode missing values. If ``'new_category'``, missing values are encoded as + their own category. If ``None``, missing values are not encoded and remain missing. + Defaults to ``'new_category'``. """ - def __init__(self, order, add_noise=False): + def __init__(self, order, add_noise=False, missing_value_encoding='new_category'): + _validate_missing_value_encoding(missing_value_encoding) + self.missing_value_encoding = missing_value_encoding self.order = pd.Series(order).fillna(np.nan) if not self.order.is_unique: err_msg = ( @@ -871,7 +976,7 @@ def __init__(self, order, add_noise=False): ) raise TransformerInputError(err_msg) - super().__init__(add_noise=add_noise) + super().__init__(add_noise=add_noise, missing_value_encoding=missing_value_encoding) def __repr__(self): """Represent initialization of transformer as text. @@ -885,10 +990,18 @@ def __repr__(self): custom_args.append('order=') if self.add_noise: custom_args.append(f'add_noise={self.add_noise}') + if self.missing_value_encoding != 'new_category': + custom_args.append(f'missing_value_encoding={repr(self.missing_value_encoding)}') args_string = ', '.join(custom_args) return f'{class_name}({args_string})' + def _get_order(self): + if self.missing_value_encoding is None: + return self.order[~pd.isna(self.order)] + + return self.order + def _fit(self, data): """Fit the transformer to the data. @@ -901,16 +1014,27 @@ def _fit(self, data): Data to fit the transformer to. """ self.dtype = data.dtype - data = data.infer_objects().fillna(np.nan) + data = data.infer_objects() + if self.missing_value_encoding == 'new_category': + data = data.fillna(np.nan) + + order = self._get_order() - missing = list(data[~data.isin(self.order)].unique()) + unknown = ~data.isin(order) + if self.missing_value_encoding is None: + unknown &= ~pd.isna(data) + + missing = list(data[unknown].unique()) if len(missing) > 0: raise TransformerInputError( f"Unknown categories '{missing}'. All possible categories must be defined in the " "'order' parameter." ) - self.values_to_categories = dict(enumerate(self.order)) + if self.missing_value_encoding is None: + data = data[~pd.isna(data)] + + self.values_to_categories = dict(enumerate(order)) self.categories_to_values = { category: value for value, category in self.values_to_categories.items() } diff --git a/rdt/transformers/numerical.py b/rdt/transformers/numerical.py index 8d23354f4..477f09cf7 100644 --- a/rdt/transformers/numerical.py +++ b/rdt/transformers/numerical.py @@ -457,7 +457,7 @@ def _reverse_transform(self, data): @property def learned_distribution(self): - """Get the learned distribution name and parameters. + """Learned distribution name and parameters. Returns: dict: diff --git a/tests/integration/transformers/test_categorical.py b/tests/integration/transformers/test_categorical.py index 85ee00a84..e528316bb 100644 --- a/tests/integration/transformers/test_categorical.py +++ b/tests/integration/transformers/test_categorical.py @@ -792,6 +792,188 @@ def test_ordered_label_encoder_numerical_nans_no_warning(): ] +@pytest.mark.parametrize( + 'transformer', + [ + UniformEncoder(missing_value_encoding=None), + OrderedUniformEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + LabelEncoder(missing_value_encoding=None), + OrderedLabelEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + ], +) +def test_categorical_transformers_missing_value_encoding_none(transformer): + """Test categorical transformers preserve missing values when configured to do so.""" + # Setup + data = pd.DataFrame({'col': [1, None, 'two', np.nan, 3, 'four']}) + + # Run + transformer.fit(data, 'col') + transformed = transformer.transform(data) + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + expected_missing_values = pd.Series([False, True, False, True, False, False], name='col') + pd.testing.assert_series_equal(transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal(reverse_transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal( + reverse_transformed.loc[~expected_missing_values, 'col'].reset_index(drop=True), + data.loc[~expected_missing_values, 'col'].reset_index(drop=True), + check_dtype=False, + ) + + +@pytest.mark.parametrize( + 'transformer', + [ + UniformEncoder(missing_value_encoding=None), + OrderedUniformEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + LabelEncoder(missing_value_encoding=None), + OrderedLabelEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + ], +) +def test_categorical_transformers_missing_value_encoding_none_when_nulls_not_seen( + transformer, +): + """Test missing values remain missing when they were not seen during fitting.""" + # Setup + fit_data = pd.DataFrame({'col': [1, 'two', 3, 'four']}) + transform_data = pd.DataFrame({'col': [1, None, 'two', np.nan, 3, 'four']}) + + # Run + transformer.fit(fit_data, 'col') + with warnings.catch_warnings(record=True) as recorded_warnings: + transformed = transformer.transform(transform_data) + + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + assert len(recorded_warnings) == 0 + expected_missing_values = pd.Series([False, True, False, True, False, False], name='col') + pd.testing.assert_series_equal(transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal(reverse_transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal( + reverse_transformed.loc[~expected_missing_values, 'col'].reset_index(drop=True), + transform_data.loc[~expected_missing_values, 'col'].reset_index(drop=True), + check_dtype=False, + ) + + +@pytest.mark.parametrize( + 'transformer', + [ + UniformEncoder(missing_value_encoding=None), + OrderedUniformEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + LabelEncoder(missing_value_encoding=None), + OrderedLabelEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + ], +) +def test_categorical_transformers_missing_value_encoding_none_all_missing(transformer): + """Test transformers with no learned categories keep missing values missing.""" + # Setup + data = pd.DataFrame({'col': pd.Series([None, np.nan, pd.NA], dtype='object')}) + + # Run + transformer.fit(data, 'col') + transformed = transformer.transform(data) + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + assert transformed['col'].isna().all() + assert reverse_transformed['col'].isna().all() + assert reverse_transformed.shape == data.shape + + +@pytest.mark.parametrize( + 'transformer', + [ + UniformEncoder(missing_value_encoding=None), + OrderedUniformEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + LabelEncoder(missing_value_encoding=None), + OrderedLabelEncoder(order=[1, 'two', 3, 'four'], missing_value_encoding=None), + ], +) +def test_categorical_transformers_missing_value_encoding_none_reverse_passes_nulls_through( + transformer, +): + """Test missing values produced downstream stay missing during reverse transform.""" + # Setup + data = pd.DataFrame({'col': [1, 'two', 3, 'four']}) + + # Run + transformer.fit(data, 'col') + transformed = transformer.transform(data) + transformed.loc[[1, 3], 'col'] = np.nan + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + expected_missing_values = pd.Series([False, True, False, True], name='col') + pd.testing.assert_series_equal(reverse_transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal( + reverse_transformed.loc[~expected_missing_values, 'col'].reset_index(drop=True), + data.loc[~expected_missing_values, 'col'].reset_index(drop=True), + check_dtype=False, + ) + + +@pytest.mark.parametrize( + 'transformer', + [ + LabelEncoder(missing_value_encoding=None), + OrderedLabelEncoder(order=['a', 'b'], missing_value_encoding=None), + ], +) +def test_label_encoders_missing_value_encoding_none_with_category_dtype(transformer): + """Test label encoders keep transformed pandas category columns numeric.""" + # Setup + data = pd.DataFrame({'col': pd.Series(['a', None, 'b'], dtype='category')}) + + # Run + transformer.fit(data, 'col') + transformed = transformer.transform(data) + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + assert pd.api.types.is_numeric_dtype(transformed['col']) + expected_missing_values = pd.Series([False, True, False], name='col') + pd.testing.assert_series_equal(transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal(reverse_transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal( + reverse_transformed.loc[~expected_missing_values, 'col'].reset_index(drop=True), + data.loc[~expected_missing_values, 'col'].reset_index(drop=True), + check_dtype=False, + ) + + +@pytest.mark.parametrize( + 'transformer', + [ + UniformEncoder(), + OrderedUniformEncoder(order=[1, 'two', 3, 'four', None]), + LabelEncoder(), + OrderedLabelEncoder(order=[1, 'two', 3, 'four', None]), + ], +) +def test_categorical_transformers_default_missing_value_encoding_new_category(transformer): + """Test default missing value handling continues to encode missing as a category.""" + # Setup + data = pd.DataFrame({'col': [1, None, 'two', np.nan, 3, 'four']}) + + # Run + transformer.fit(data, 'col') + transformed = transformer.transform(data) + reverse_transformed = transformer.reverse_transform(transformed) + + # Assert + assert transformed['col'].notna().all() + expected_missing_values = pd.Series([False, True, False, True, False, False], name='col') + pd.testing.assert_series_equal(reverse_transformed['col'].isna(), expected_missing_values) + pd.testing.assert_series_equal( + reverse_transformed.loc[~expected_missing_values, 'col'].reset_index(drop=True), + data.loc[~expected_missing_values, 'col'].reset_index(drop=True), + check_dtype=False, + ) + + @pytest.mark.parametrize('sdtype', ['id', 'text']) @pytest.mark.parametrize('transformer', categorical_transformers) def test_categorical_transformers_with_id_sdtype(sdtype, transformer): diff --git a/tests/unit/transformers/test_categorical.py b/tests/unit/transformers/test_categorical.py index f084d782f..5eeb17d83 100644 --- a/tests/unit/transformers/test_categorical.py +++ b/tests/unit/transformers/test_categorical.py @@ -39,6 +39,15 @@ def test___init___bad_order_by(self): with pytest.raises(TransformerInputError, match=message): UniformEncoder(order_by='bad_value') + def test___init___bad_missing_value_encoding(self): + """Test that the ``__init__`` raises error if ``missing_value_encoding`` is invalid.""" + # Run / Assert + message = ( + "'missing_value_encoding' must be one of the following values: None or 'new_category'." + ) + with pytest.raises(TransformerInputError, match=message): + UniformEncoder(missing_value_encoding='bad_value') + def test__order_categories_alphabetical(self): """Test the ``_order_categories`` method when ``order_by`` is 'alphabetical'. @@ -216,6 +225,21 @@ def test_fit_with_nullable_integer_dtype(self): } assert transformer.frequencies == expected_frequencies + def test__fit_missing_value_encoding_none(self): + """Test that missing values are ignored during fit when configured.""" + # Setup + data = pd.Series(['foo', None, 'bar', np.nan]) + transformer = UniformEncoder(missing_value_encoding=None) + + # Run + transformer._fit(data) + + # Assert + expected_frequencies = {'foo': 0.5, 'bar': 0.5} + expected_intervals = {'foo': [0.0, 0.5], 'bar': [0.5, 1.0]} + assert transformer.frequencies == expected_frequencies + assert transformer.intervals == expected_intervals + def test__set_fitted_parameters(self): """Test the ``_set_fitted_parameters`` method.""" # Setup @@ -263,6 +287,24 @@ def test__transform(self): assert (transformed.loc[data == key] >= transformer.intervals[key][0]).all() assert (transformed.loc[data == key] < transformer.intervals[key][1]).all() + def test__transform_missing_value_encoding_none(self): + """Test missing values are not encoded during transform when configured.""" + # Setup + transformer = UniformEncoder(missing_value_encoding=None) + data = pd.Series(['foo', None, 'bar', np.nan]) + transformer.frequencies = {'foo': 0.5, 'bar': 0.5} + transformer.intervals = {'foo': [0.0, 0.5], 'bar': [0.5, 1.0]} + + # Run + transformed = transformer._transform(data) + + # Assert + assert transformed.loc[[1, 3]].isna().all() + assert (transformed.loc[data == 'foo'] >= transformer.intervals['foo'][0]).all() + assert (transformed.loc[data == 'foo'] < transformer.intervals['foo'][1]).all() + assert (transformed.loc[data == 'bar'] >= transformer.intervals['bar'][0]).all() + assert (transformed.loc[data == 'bar'] < transformer.intervals['bar'][1]).all() + def test__transform_user_warning(self): """Test the ``transform`` with unknown data. @@ -424,6 +466,21 @@ def test__reverse_transform_integer_and_nans(self): # Assert pd.testing.assert_series_equal(out, pd.Series([11, 12, np.nan, 13])) + def test__reverse_transform_empty_intervals(self): + """Test ``_reverse_transform``with nothing learned.""" + # Setup + transformer = UniformEncoder(missing_value_encoding=None) + transformer.intervals = {} + transformer.dtype = 'object' + data = pd.Series([0.1, np.nan], name='column_name') + + # Run + out = transformer._reverse_transform(data) + + # Assert + expected = pd.Series([np.nan, np.nan], name='column_name') + pd.testing.assert_series_equal(out, expected, check_dtype=False) + @pytest.fixture(autouse=True) def _setup_caplog(caplog): @@ -472,6 +529,18 @@ def test___repr___default(self): # Assert assert stringified_transformer == 'OrderedUniformEncoder(order=)' + def test___repr___missing_value_encoding_none(self): + """Test that the ``__repr__`` method prints non-default missing value encoding.""" + # Setup + transformer = OrderedUniformEncoder(order=['VISA', 'AMEX'], missing_value_encoding=None) + + # Run + stringified_transformer = transformer.__repr__() + + # Assert + expected = 'OrderedUniformEncoder(order=, missing_value_encoding=None)' + assert stringified_transformer == expected + def test__fit(self): """Test the ``_fit`` method.""" # Setup @@ -596,6 +665,19 @@ def test__transform_error(self): with pytest.raises(TransformerInputError, match=message): transformer._transform(data) + def test__fit_missing_value_encoding_none_order_without_missing_values(self): + """Test missing values are allowed outside the order when not encoded.""" + # Setup + data = pd.Series(['a', None, 'b', np.nan]) + transformer = OrderedUniformEncoder(order=['b', 'a'], missing_value_encoding=None) + + # Run + transformer._fit(data) + + # Assert + assert transformer.frequencies == {'b': 0.5, 'a': 0.5} + assert transformer.intervals == {'b': [0.0, 0.5], 'a': [0.5, 1.0]} + class TestFrequencyEncoder: def test___setstate__(self): @@ -1946,6 +2028,15 @@ def test___init___bad_order_by(self): with pytest.raises(TransformerInputError, match=message): LabelEncoder(order_by='bad_value') + def test___init___bad_missing_value_encoding(self): + """Test that the ``__init__`` raises error if ``missing_value_encoding`` is invalid.""" + # Run / Assert + message = ( + "'missing_value_encoding' must be one of the following values: None or 'new_category'." + ) + with pytest.raises(TransformerInputError, match=message): + LabelEncoder(missing_value_encoding='bad_value') + def test__order_categories_alphabetical(self): """Test the ``_order_categories`` method when ``order_by`` is 'alphabetical'. @@ -2085,6 +2176,18 @@ def test__order_categories_numerical_different_dtype_error(self): with pytest.raises(TransformerInputError, match=message): transformer._order_categories(arr) + def test__order_categories_empty(self): + """Test the ``_order_categories`` method with empty data.""" + # Setup + transformer = LabelEncoder() + unique_data = np.array([]) + + # Run + ordered = transformer._order_categories(unique_data) + + # Assert + np.testing.assert_array_equal(ordered, unique_data) + def test__fit(self): """Test the ``_fit`` method. @@ -2116,6 +2219,19 @@ def test__fit(self): None: {'sdtype': 'float', 'next_transformer': None}, } + def test__fit_missing_value_encoding_none(self): + """Test that missing values are ignored during fit when configured.""" + # Setup + data = pd.Series(['foo', None, 'bar', np.nan]) + transformer = LabelEncoder(missing_value_encoding=None) + + # Run + transformer._fit(data) + + # Assert + assert transformer.values_to_categories == {0: 'foo', 1: 'bar'} + assert transformer.categories_to_values == {'foo': 0, 'bar': 1} + def test__transform(self): """Test the ``_transform`` method. @@ -2154,6 +2270,21 @@ def test__transform(self): assert 0 <= transformed[3] <= 2 + def test__transform_missing_value_encoding_none(self): + """Test missing values are not encoded during transform when configured.""" + # Setup + data = pd.Series(['foo', None, 'bar', np.nan]) + transformer = LabelEncoder(missing_value_encoding=None) + transformer.categories_to_values = {'foo': 0, 'bar': 1} + transformer.values_to_categories = {0: 'foo', 1: 'bar'} + + # Run + transformed = transformer._transform(data) + + # Assert + expected = pd.Series([0.0, np.nan, 1.0, np.nan]) + pd.testing.assert_series_equal(transformed, expected) + def test__transform_add_noise(self): """Test the ``_transform`` method with ``add_noise``. @@ -2251,6 +2382,21 @@ def test__reverse_transform_clips_values(self): # Assert pd.testing.assert_series_equal(out, pd.Series(['a', 'b', 'c'])) + def test__reverse_transform_empty_values_to_categories(self): + """Test the ``_reverse_transform`` method when nothing was learned.""" + # Setup + transformer = LabelEncoder(missing_value_encoding=None) + transformer.values_to_categories = {} + transformer.dtype = 'object' + data = pd.Series([0.0, np.nan], name='column_name') + + # Run + out = transformer._reverse_transform(data) + + # Assert + expected = pd.Series([np.nan, np.nan], name='column_name') + pd.testing.assert_series_equal(out, expected, check_dtype=False) + @patch('rdt.transformers.categorical.check_nan_in_transform') @patch('rdt.transformers.categorical.try_convert_to_dtype') def test__reverse_transform_add_noise(self, mock_convert_dtype, mock_check_nan): @@ -2355,6 +2501,18 @@ def test___repr___add_noise_true(self): # Assert assert stringified_transformer == 'OrderedLabelEncoder(order=, add_noise=True)' + def test___repr___missing_value_encoding_none(self): + """Test that the ``__repr__`` method prints non-default missing value encoding.""" + # Setup + transformer = OrderedLabelEncoder(order=['VISA', 'AMEX'], missing_value_encoding=None) + + # Run + stringified_transformer = transformer.__repr__() + + # Assert + expected = 'OrderedLabelEncoder(order=, missing_value_encoding=None)' + assert stringified_transformer == expected + def test__fit(self): """Test the ``_fit`` method. @@ -2406,6 +2564,19 @@ def test__fit_error(self): with pytest.raises(TransformerInputError, match=message): transformer._fit(data) + def test__fit_missing_value_encoding_none_order_without_missing_values(self): + """Test missing values are allowed outside the order when not encoded.""" + # Setup + data = pd.Series(['a', None, 'b', np.nan]) + transformer = OrderedLabelEncoder(order=['b', 'a'], missing_value_encoding=None) + + # Run + transformer._fit(data) + + # Assert + assert transformer.values_to_categories == {0: 'b', 1: 'a'} + assert transformer.categories_to_values == {'b': 0, 'a': 1} + class TestCustomLabelEncoder: def test___init__(self):