From ce294a24a23770b15ec9ffdc4047fbf98713e1c9 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 26 Aug 2026 00:58:33 +0200 Subject: [PATCH 1/7] Migrate Winsoriser/Winsorizer to narwhals, add polars support Removed the module-level `import pandas as pd` and `import numpy as np`; X type hints now use narwhals' IntoDataFrame. WinsorizerBase.fit/transform (shared base) were already migrated on origin/narwhals-outliers-base; this change covers the Winsoriser-specific piece: transform()'s add_indicators path, which compares the capped output against the original input to build per-tail boolean flag columns and previously only worked on pandas. Benchmarked the add_indicators comparison+concat step at 10k/50k/100k rows x 1/2/10 columns: pandas-native (boolean comparison + pd.concat) is up to ~3x faster than the narwhals with_columns equivalent on pandas input, and the loss grows with column count (1 col: narwhals-on-pandas was actually faster; 10 cols: ~2-3x slower). That crosses the "keep pandas fast path" threshold, so transform() splits on `nwd.is_pandas_dataframe`, matching MissingIndicator's precedent for its own indicator-building step: pandas keeps its existing comparison+concat logic (now obtaining the `pd` module via `nw.from_native(...).__native_namespace__()` instead of importing it), and a new narwhals with_columns path (per-column Series comparison, cast to Float64) covers polars and other backends. Preserved the Winsoriser/Winsorizer deprecation exactly as-is: Winsoriser is the current public name (renamed to the British spelling in #967); Winsorizer is a deprecated subclass that raises the same FutureWarning on __init__ and will be removed in 2.1.0. Note this is the reverse of what one might guess from the class names alone. Tests: converted tests/test_outliers/test_winsorizer.py from pandas-only fixtures (df_normal_dist, df_vartypes, df_na) to local dicts parametrized over `make_df` in [pd.DataFrame, pl.DataFrame], asserting identical capping values, indicator columns, and get_feature_names_out() on both backends for the same input. Missing-value dicts use None instead of np.nan in string columns, since polars' DataFrame constructor rejects a float NaN mixed into a string column. A helper filters both pandas' NaN and polars' None representations of a missing value when comparing outputs cross-backend. Docs: verified every doc example in docs/user_guide/outliers/Winsoriser.rst against actual output (network access to fetch_openml's house_prices dataset was available; outputs matched exactly, no changes needed) and added a "With polars" section covering add_indicators, matching the pattern used in other migrated user guides. Added a verified "With polars" example to the class docstring. Verified: tests/test_outliers/test_winsorizer.py 93 passed. Full tests/test_outliers suite: 123 passed / 3 pre-existing failures in test_check_estimator_outliers.py (confirmed identical against a baseline run of origin/narwhals-outliers-base: 83 passed / same 3 failures - sklearn's check_estimator feeds raw numpy arrays, which check_X() has always rejected per the narwhals migration's dataframe-only contract; predates this change). flake8 and mypy clean. sphinx -W build clean (only the pre-existing unrelated linkcode_resolve warning, confirmed present on the base branch too). Confirmed winsorizer.py and base_outlier.py import successfully and a full polars fit_transform (including add_indicators) runs correctly with pandas' own import blocked at the builtins level. Co-Authored-By: Claude Sonnet 5 --- docs/user_guide/outliers/Winsoriser.rst | 56 ++++ feature_engine/outliers/winsorizer.py | 119 +++++--- tests/test_outliers/test_winsorizer.py | 360 +++++++++++++----------- 3 files changed, 331 insertions(+), 204 deletions(-) diff --git a/docs/user_guide/outliers/Winsoriser.rst b/docs/user_guide/outliers/Winsoriser.rst index 31babf233..cf6fbee2d 100644 --- a/docs/user_guide/outliers/Winsoriser.rst +++ b/docs/user_guide/outliers/Winsoriser.rst @@ -354,6 +354,62 @@ The default values for fold are as follows: You can manually adjust the `fold` value to make the outlier detection process more or less conservative, thus customising the extent of outlier capping. +With polars +----------- + +:class:`Winsoriser()` works in the same way with a polars dataframe, including the +`add_indicators` option, which flags the rows that were capped on each tail: + +.. code:: python + + import polars as pl + from feature_engine.outliers import Winsoriser + + df = pl.DataFrame({ + "Age": [20, 21, 19, 18, 23, 40, 41, 97], + "Marks": [0.9, 0.8, 0.7, 0.6, 0.3, 0.5, 0.8, 0.05], + }) + + transformer = Winsoriser( + capping_method="iqr", tail="both", fold=1.5, add_indicators=True, + ) + transformer.fit(df) + + print(transformer.right_tail_caps_) + print(transformer.left_tail_caps_) + +The learned capping values match those found with pandas: + +.. code:: text + + {'Age': 71.0, 'Marks': 1.3250000000000002} + {'Age': -11.0, 'Marks': -0.07500000000000001} + +.. code:: python + + print(transformer.transform(df)) + +`Age`'s outlier, 97, was capped to 71 and flagged in `Age_right`; none of the values +in `Marks` were extreme enough to be capped: + +.. code:: text + + shape: (8, 6) + ┌──────┬───────┬──────────┬───────────┬────────────┬─────────────┐ + │ Age ┆ Marks ┆ Age_left ┆ Age_right ┆ Marks_left ┆ Marks_right │ + │ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │ + │ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 ┆ f64 │ + ╞══════╪═══════╪══════════╪═══════════╪════════════╪═════════════╡ + │ 20.0 ┆ 0.9 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 21.0 ┆ 0.8 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 19.0 ┆ 0.7 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 18.0 ┆ 0.6 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 23.0 ┆ 0.3 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 40.0 ┆ 0.5 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 41.0 ┆ 0.8 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ + │ 71.0 ┆ 0.05 ┆ 0.0 ┆ 1.0 ┆ 0.0 ┆ 0.0 │ + └──────┴───────┴──────────┴───────────┴────────────┴─────────────┘ + Additional resources -------------------- diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index ad2320ab9..1a4fa5194 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -4,8 +4,9 @@ import warnings from typing import List, Literal, Union -import numpy as np -import pandas as pd +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame from feature_engine._docstrings.fit_attributes import ( _feature_names_in_docstring, @@ -145,25 +146,33 @@ class Winsoriser(WinsorizerBase): 8 -0.469474 9 0.542560 + With polars: + >>> import numpy as np - >>> import pandas as pd + >>> import polars as pl >>> from feature_engine.outliers import Winsoriser >>> np.random.seed(42) - >>> X = pd.DataFrame(dict(x = np.random.normal(size = 10))) + >>> X = pl.DataFrame(dict(x = np.random.normal(size = 10))) >>> wz = Winsoriser(capping_method='mad', tail='both', fold=3) >>> wz.fit(X) >>> wz.transform(X) - x - 0 0.496714 - 1 -0.138264 - 2 0.647689 - 3 1.523030 - 4 -0.234153 - 5 -0.234137 - 6 1.579213 - 7 0.767435 - 8 -0.469474 - 9 0.542560 + shape: (10, 1) + ┌───────────┐ + │ x │ + │ --- │ + │ f64 │ + ╞═══════════╡ + │ 0.496714 │ + │ -0.138264 │ + │ 0.647689 │ + │ 1.52303 │ + │ -0.234153 │ + │ -0.234137 │ + │ 1.579213 │ + │ 0.767435 │ + │ -0.469474 │ + │ 0.54256 │ + └───────────┘ """ def __init__( @@ -186,18 +195,18 @@ def __init__( ) self.add_indicators = add_indicators - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Cap the variable values. Optionally, add outlier indicators. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe of shape = [n_samples, n_features + n_ind] + X_new: dataframe of shape = [n_samples, n_features + n_ind] The dataframe with the capped variables and indicators. The number of output variables depends on the values for 'tail' and 'add_indicators': if passing 'add_indicators=False', will be equal @@ -210,29 +219,59 @@ def transform(self, X: pd.DataFrame) -> pd.DataFrame: else: X_orig = check_X(X) X_out = super()._transform(X_orig) - X_orig = X_orig[self.variables_] - X_out_filtered = X_out[self.variables_] - - if self.tail in ["left", "both"]: - X_left = X_out_filtered > X_orig - X_left.columns = [str(cl) + "_left" for cl in self.variables_] - if self.tail in ["right", "both"]: - X_right = X_out_filtered < X_orig - X_right.columns = [str(cl) + "_right" for cl in self.variables_] - if self.tail == "left": - X_out = pd.concat([X_out, X_left.astype(np.float64)], axis=1) - elif self.tail == "right": - X_out = pd.concat([X_out, X_right.astype(np.float64)], axis=1) - else: - X_both = pd.concat([X_left, X_right], axis=1).astype(np.float64) - X_both = X_both[ - [ - cl1 - for cl2 in zip(X_left.columns.values, X_right.columns.values) - for cl1 in cl2 + + # Benchmarked at 10k-100k rows x 1-10 columns: pandas-native + # comparison + concat is up to ~3x faster than the narwhals + # with_columns equivalent on pandas input (the loss grows with + # column count), so pandas keeps its own fast path here, same + # split as MissingIndicator's indicator-building step. + is_pandas = nwd.is_pandas_dataframe(X_out) + if is_pandas is True: + pd = nw.from_native(X_out, eager_only=True).__native_namespace__() + X_orig_filtered = X_orig[self.variables_] + X_out_filtered = X_out[self.variables_] + + if self.tail in ["left", "both"]: + X_left = X_out_filtered > X_orig_filtered + X_left.columns = [str(cl) + "_left" for cl in self.variables_] + if self.tail in ["right", "both"]: + X_right = X_out_filtered < X_orig_filtered + X_right.columns = [str(cl) + "_right" for cl in self.variables_] + if self.tail == "left": + X_out = pd.concat([X_out, X_left.astype("float64")], axis=1) + elif self.tail == "right": + X_out = pd.concat([X_out, X_right.astype("float64")], axis=1) + else: + X_both = pd.concat([X_left, X_right], axis=1).astype("float64") + X_both = X_both[ + [ + cl1 + for cl2 in zip( + X_left.columns.values, X_right.columns.values + ) + for cl1 in cl2 + ] ] - ] - X_out = pd.concat([X_out, X_both], axis=1) + X_out = pd.concat([X_out, X_both], axis=1) + else: + nw_orig = nw.from_native(X_orig, eager_only=True) + nw_out = nw.from_native(X_out, eager_only=True) + + new_cols = [] + for var in self.variables_: + if self.tail in ["left", "both"]: + new_cols.append( + (nw_out[var] > nw_orig[var]) + .cast(nw.Float64) + .alias(f"{var}_left") + ) + if self.tail in ["right", "both"]: + new_cols.append( + (nw_out[var] < nw_orig[var]) + .cast(nw.Float64) + .alias(f"{var}_right") + ) + X_out = nw_out.with_columns(*new_cols).to_native() return X_out diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index 1264bece3..e0fe62346 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -1,8 +1,10 @@ import math import re +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.outliers import Winsoriser, Winsorizer @@ -12,6 +14,66 @@ "be removed in version 2.1.0. To silence this warning, use Winsoriser instead." ) +# mirrors tests/conftest.py's df_vartypes fixture, minus the datetime "dob" +# column (values that don't round-trip identically between pandas and +# polars are irrelevant here, since Winsoriser only touches numerical vars) +VARTYPES = { + "Name": ["tom", "nick", "krish", "jack"], + "City": ["London", "Manchester", "Liverpool", "Bristol"], + "Age": [20, 21, 19, 18], + "Marks": [0.9, 0.8, 0.7, 0.6], +} + +# mirrors tests/conftest.py's df_na fixture, minus "dob"; uses None (not +# np.nan) for missing values in string columns since polars' DataFrame +# constructor rejects a float NaN mixed into a string column +DATA_NA = { + "Name": ["tom", "nick", "krish", None, "peter", None, "fred", "sam"], + "City": [ + "London", + "Manchester", + None, + None, + "London", + "London", + "Bristol", + "Manchester", + ], + "Studies": [ + "Bachelor", + "Bachelor", + None, + None, + "Bachelor", + "PhD", + "None", + "Masters", + ], + "Age": [20, 21, 19, None, 23, 40, 41, 37], + "Marks": [0.9, 0.8, 0.7, None, 0.3, None, 0.8, 0.6], +} + + +def _col(X, col): + return nw.from_native(X, eager_only=True)[col].to_list() + + +def _cols(X): + return list(nw.from_native(X, eager_only=True).columns) + + +def _shape(X): + return nw.from_native(X, eager_only=True).shape + + +def _drop_missing(values): + # pandas yields float('nan') for a missing numeric value; polars yields + # None. Filter both so tests can assert on the same non-missing values + # regardless of backend. + return [ + v for v in values if v is not None and not (isinstance(v, float) and v != v) + ] + @pytest.fixture( params=[Winsoriser, Winsorizer], @@ -33,16 +95,14 @@ def test_winsorizer_raises_future_warning(): Winsorizer() -def test_gaussian_capping_right_tail_with_fold_1(df_normal_dist, transformer_class): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_gaussian_capping_right_tail_with_fold_1(make_df, transformer_class): # test case 1: mean and std, right tail + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="gaussian", tail="right", fold=1 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(upper=0.1067690260251065) + X_out = transformer.fit_transform(X) # test init params assert transformer.capping_method == "gaussian" @@ -53,143 +113,117 @@ def test_gaussian_capping_right_tail_with_fold_1(df_normal_dist, transformer_cla assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform outputs - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.10676902602510658) - assert math.isclose(df_transf["var"].max(), 0.1067690260251065) + assert math.isclose(max(_col(X_out, "var")), 0.1067690260251065) -def test_gaussian_capping_both_tails_with_fold_2(df_normal_dist, transformer_class): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_gaussian_capping_both_tails_with_fold_2(make_df, transformer_class): # test case 2: mean and std, both tails, different fold value + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="gaussian", tail="both", fold=2 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(-0.1955956473898675, 0.2075572504967645) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.2075572504967645) assert math.isclose(transformer.left_tail_caps_["var"], -0.1955956473898675) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.2075572504967645) - assert math.isclose(X["var"].min(), -0.1955956473898675) - assert math.isclose(df_transf["var"].max(), 0.2075572504967645) - assert math.isclose(df_transf["var"].min(), -0.1955956473898675) + values = _col(X_out, "var") + assert math.isclose(max(values), 0.2075572504967645) + assert math.isclose(min(values), -0.1955956473898675) -def test_iqr_capping_both_tails_with_fold_1(df_normal_dist, transformer_class): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_iqr_capping_both_tails_with_fold_1(make_df, transformer_class): # test case 3: IQR, both tails, fold 1 + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="iqr", tail="both", fold=1 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(-0.20247907173293223, 0.21180113880445128) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.21180113880445128) assert math.isclose(transformer.left_tail_caps_["var"], -0.20247907173293223) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.21180113880445128) - assert math.isclose(X["var"].min(), -0.20247907173293223) - assert math.isclose(df_transf["var"].max(), 0.21180113880445128) - assert math.isclose(df_transf["var"].min(), -0.20247907173293223) + values = _col(X_out, "var") + assert math.isclose(max(values), 0.21180113880445128) + assert math.isclose(min(values), -0.20247907173293223) -def test_iqr_capping_left_tail_with_fold_2(df_normal_dist, transformer_class): - # test case 4: IQR, left tail, fold 2 +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_iqr_capping_left_tail_with_fold_2(make_df, transformer_class): + # test case 4: IQR, left tail, fold 0.8 + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="iqr", tail="left", fold=0.8 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(lower=-0.17486039103044) + X_out = transformer.fit_transform(X) # test fit params assert transformer.right_tail_caps_ == {} assert math.isclose(transformer.left_tail_caps_["var"], -0.17486039103044) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].min(), -0.17486039103044) - assert math.isclose(df_transf["var"].min(), -0.17486039103044) + assert math.isclose(min(_col(X_out, "var")), -0.17486039103044) -def test_quantile_capping_both_tails_with_fold_10_percent( - df_normal_dist, transformer_class -): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_quantile_capping_both_tails_with_fold_10_percent(make_df, transformer_class): # test case 5: quantiles, both tails, fold 10% + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="quantiles", tail="both", fold=0.1 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(-0.12366227743232801, 0.14712481122898166) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.14712481122898166) assert math.isclose(transformer.left_tail_caps_["var"], -0.12366227743232801) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.14712481122898166) - assert math.isclose(X["var"].min(), -0.12366227743232801) - assert math.isclose(df_transf["var"].max(), 0.14712481122898166) - assert math.isclose(df_transf["var"].min(), -0.12366227743232801) + values = _col(X_out, "var") + assert math.isclose(max(values), 0.14712481122898166) + assert math.isclose(min(values), -0.12366227743232801) -def test_quantile_capping_right_tail_with_fold_15_percent( - df_normal_dist, transformer_class -): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_quantile_capping_right_tail_with_fold_15_percent(make_df, transformer_class): # test case 6: quantiles, right tail, fold 15% + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="quantiles", tail="right", fold=0.15 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(upper=0.11823196128033647) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.11823196128033647) assert transformer.left_tail_caps_ == {} # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.11823196128033647) - assert math.isclose(df_transf["var"].max(), 0.11823196128033647) + assert math.isclose(max(_col(X_out, "var")), 0.11823196128033647) +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "strings,expected", [("gaussian", 3), ("iqr", 1.5), ("mad", 3.29), ("quantiles", 0.05)], ) -def test_auto_fold_default_value(strings, expected, df_normal_dist, transformer_class): +def test_auto_fold_default_value(make_df, strings, expected, transformer_class): + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method=strings, fold="auto" ) - transformer.fit(df_normal_dist) + transformer.fit(X) assert transformer.fold_ == expected -def test_mad_capping_right_tail_with_fold_1(df_normal_dist, transformer_class): - # test case 1: median and mad, right tail +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_mad_capping_right_tail_with_fold_1(make_df, transformer_class): + # test case: median and mad, right tail + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="mad", tail="right", fold=1 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(upper=0.10995521088494983) + X_out = transformer.fit_transform(X) # test init params assert transformer.capping_method == "mad" @@ -200,34 +234,32 @@ def test_mad_capping_right_tail_with_fold_1(df_normal_dist, transformer_class): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform outputs - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.10995521088494983) - assert math.isclose(df_transf["var"].max(), 0.10995521088494983) + assert math.isclose(max(_col(X_out, "var")), 0.10995521088494983) -def test_mad_capping_both_tails_with_fold_2(df_normal_dist, transformer_class): - # test case 2: mean and std, both tails, different fold value +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_mad_capping_both_tails_with_fold_2(make_df, transformer_class): + # test case: mad, both tails, different fold value + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="mad", tail="both", fold=2 ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = df_transf["var"].clip(-0.1916815859385002, 0.21050080982609987) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.21050080982609987) assert math.isclose(transformer.left_tail_caps_["var"], -0.1916815859385002) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["var"].max(), 0.21050080982609987) - assert math.isclose(X["var"].min(), -0.1916815859385002) - assert math.isclose(df_transf["var"].max(), 0.21050080982609987) - assert math.isclose(df_transf["var"].min(), -0.1916815859385002) + values = _col(X_out, "var") + assert math.isclose(max(values), 0.21050080982609987) + assert math.isclose(min(values), -0.1916815859385002) -def test_indicators_are_added(df_normal_dist, transformer_class): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_indicators_are_added(make_df, transformer_class): + X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) + n_cols = _shape(X)[1] + transformer = make_transformer( transformer_class, tail="both", @@ -235,10 +267,10 @@ def test_indicators_are_added(df_normal_dist, transformer_class): fold=0.1, add_indicators=True, ) - X = transformer.fit_transform(df_normal_dist) - # test that the number of output variables is correct - assert X.shape[1] == 3 * df_normal_dist.shape[1] - assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == 3 * n_cols + for col in _cols(X_out)[n_cols:]: + assert sum(_col(X_out, col)) > 0 transformer = make_transformer( transformer_class, @@ -247,9 +279,10 @@ def test_indicators_are_added(df_normal_dist, transformer_class): fold=0.1, add_indicators=True, ) - X = transformer.fit_transform(df_normal_dist) - assert X.shape[1] == 2 * df_normal_dist.shape[1] - assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == 2 * n_cols + for col in _cols(X_out)[n_cols:]: + assert sum(_col(X_out, col)) > 0 transformer = make_transformer( transformer_class, @@ -258,12 +291,17 @@ def test_indicators_are_added(df_normal_dist, transformer_class): fold=0.1, add_indicators=True, ) - X = transformer.fit_transform(df_normal_dist) - assert X.shape[1] == 2 * df_normal_dist.shape[1] - assert np.all(X.iloc[:, df_normal_dist.shape[1]:].sum(axis=0) > 0) + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == 2 * n_cols + for col in _cols(X_out)[n_cols:]: + assert sum(_col(X_out, col)) > 0 + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_indicators_filter_variables(make_df, transformer_class): + X = make_df(VARTYPES) + n_cols = _shape(X)[1] -def test_indicators_filter_variables(df_vartypes, transformer_class): transformer = make_transformer( transformer_class, variables=["Age", "Marks"], @@ -272,19 +310,24 @@ def test_indicators_filter_variables(df_vartypes, transformer_class): fold=0.1, add_indicators=True, ) - X = transformer.fit_transform(df_vartypes) - assert X.shape[1] == df_vartypes.shape[1] + 4 + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == n_cols + 4 transformer.set_params(tail="left") - X = transformer.fit_transform(df_vartypes) - assert X.shape[1] == df_vartypes.shape[1] + 2 + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == n_cols + 2 transformer.set_params(tail="right") - X = transformer.fit_transform(df_vartypes) - assert X.shape[1] == df_vartypes.shape[1] + 2 + X_out = transformer.fit_transform(X) + assert _shape(X_out)[1] == n_cols + 2 + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_indicators_are_correct(make_df, transformer_class): + X = make_df({"col": np.arange(100).astype(np.float64)}) + expected_left = [1.0] * 10 + [0.0] * 90 + expected_right = [0.0] * 90 + [1.0] * 10 -def test_indicators_are_correct(transformer_class): transformer = make_transformer( transformer_class, tail="left", @@ -292,39 +335,24 @@ def test_indicators_are_correct(transformer_class): fold=0.1, add_indicators=True, ) - df = pd.DataFrame({"col": np.arange(100).astype(np.float64)}) - df_out = transformer.fit_transform(df) - expected_ind = np.r_[np.repeat(True, 10), np.repeat(False, 90)].astype(np.float64) - pd.testing.assert_frame_equal( - df_out.drop("col", axis=1), df.assign(col_left=expected_ind).drop("col", axis=1) - ) + X_out = transformer.fit_transform(X) + assert _col(X_out, "col_left") == expected_left transformer.set_params(tail="right") - df_out = transformer.fit_transform(df) - expected_ind = np.r_[np.repeat(False, 90), np.repeat(True, 10)].astype(np.float64) - pd.testing.assert_frame_equal( - df_out.drop("col", axis=1), - df.assign(col_right=expected_ind).drop("col", axis=1), - ) + X_out = transformer.fit_transform(X) + assert _col(X_out, "col_right") == expected_right transformer.set_params(tail="both") - df_out = transformer.fit_transform(df) - expected_ind_left = np.r_[np.repeat(True, 10), np.repeat(False, 90)].astype( - np.float64 - ) - expected_ind_right = np.r_[np.repeat(False, 90), np.repeat(True, 10)].astype( - np.float64 - ) - pd.testing.assert_frame_equal( - df_out.drop("col", axis=1), - df.assign(col_left=expected_ind_left, col_right=expected_ind_right).drop( - "col", axis=1 - ), - ) + X_out = transformer.fit_transform(X) + assert _col(X_out, "col_left") == expected_left + assert _col(X_out, "col_right") == expected_right + assert _cols(X_out) == ["col", "col_left", "col_right"] -def test_transformer_ignores_na_in_df(df_na, transformer_class): - # test case 7: dataset contains na and transformer is asked to ignore them +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transformer_ignores_na_in_df(make_df, transformer_class): + # test case: dataset contains na and transformer is asked to ignore them + X = make_df(DATA_NA) transformer = make_transformer( transformer_class, capping_method="gaussian", @@ -333,28 +361,21 @@ def test_transformer_ignores_na_in_df(df_na, transformer_class): variables=["Age", "Marks"], missing_values="ignore", ) - X = transformer.fit_transform(df_na) - - # expected output - df_transf = df_na.copy() - df_transf["Age"] = df_transf["Age"].clip(upper=38.04494616731882) - df_transf["Marks"] = df_transf["Marks"].clip(upper=0.8784116651786605) + X_out = transformer.fit_transform(X) # test fit params assert math.isclose(transformer.right_tail_caps_["Age"], 38.04494616731882) assert math.isclose(transformer.right_tail_caps_["Marks"], 0.8784116651786605) assert transformer.left_tail_caps_ == {} - assert transformer.n_features_in_ == 6 + assert transformer.n_features_in_ == 5 # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert math.isclose(X["Age"].max(), 38.04494616731882) - assert math.isclose(X["Age"].max(), 38.04494616731882) - assert math.isclose(X["Marks"].max(), 0.8784116651786605) - assert math.isclose(df_transf["Marks"].max(), 0.8784116651786605) + age = _drop_missing(_col(X_out, "Age")) + marks = _drop_missing(_col(X_out, "Marks")) + assert math.isclose(max(age), 38.04494616731882) + assert math.isclose(max(marks), 0.8784116651786605) def test_error_if_capping_method_not_permitted(transformer_class): - # test error raises with pytest.raises(ValueError): make_transformer(transformer_class, capping_method="other") @@ -390,25 +411,32 @@ def test_error_if_add_incators_not_permitted(transformer_class): make_transformer(transformer_class, add_indicators=[True]) -def test_fit_raises_error_if_na_in_inut_df(df_na, transformer_class): - # test case 8: when dataset contains na, fit method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_raises_error_if_na_in_inut_df(make_df, transformer_class): + # test case: when dataset contains na, fit method + X = make_df(DATA_NA) with pytest.raises(ValueError): transformer = make_transformer(transformer_class) - transformer.fit(df_na) + transformer.fit(X) -def test_transform_raises_error_if_na_in_input_df( - df_vartypes, df_na, transformer_class -): - # test case 9: when dataset contains na, transform method +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_transform_raises_error_if_na_in_input_df(make_df, transformer_class): + # test case: when dataset contains na, transform method + X_fit = make_df(VARTYPES) + X_na = make_df( + {k: DATA_NA[k] for k in ["Name", "City", "Age", "Marks"]} + ) with pytest.raises(ValueError): transformer = make_transformer(transformer_class) - transformer.fit(df_vartypes) - transformer.transform(df_na[["Name", "City", "Age", "Marks", "dob"]]) + transformer.fit(X_fit) + transformer.transform(X_na) -def test_get_feature_names_out(df_na, transformer_class): - original_features = df_na.columns.to_list() +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_get_feature_names_out(make_df, transformer_class): + X = make_df(DATA_NA) + original_features = _cols(X) input_features = ["Age", "Marks"] # when indicators is false, we've got the generic check. @@ -419,7 +447,7 @@ def test_get_feature_names_out(df_na, transformer_class): add_indicators=True, missing_values="ignore", ) - tr.fit(df_na) + tr.fit(X) out = [f + "_left" for f in input_features] assert tr.get_feature_names_out() == original_features + out @@ -431,7 +459,7 @@ def test_get_feature_names_out(df_na, transformer_class): add_indicators=True, missing_values="ignore", ) - tr.fit(df_na) + tr.fit(X) out = [f + "_right" for f in input_features] assert tr.get_feature_names_out() == original_features + out @@ -443,14 +471,18 @@ def test_get_feature_names_out(df_na, transformer_class): add_indicators=True, missing_values="ignore", ) - tr.fit(df_na) + tr.fit(X) out = ["Age_left", "Age_right", "Marks_left", "Marks_right"] assert tr.get_feature_names_out() == original_features + out assert tr.get_feature_names_out(original_features) == original_features + out -def test_low_variation(df_normal_dist, transformer_class): +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_low_variation(make_df, transformer_class): + X = make_df( + {"var": (np.random.RandomState(0).normal(0, 0.1, 100) // 10).tolist()} + ) transformer = make_transformer(transformer_class, capping_method="mad") with pytest.raises(ValueError): - transformer.fit(df_normal_dist // 10) + transformer.fit(X) From cac5d8605747ae4559bf4538515c7bc356ddaad3 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:37:56 +0200 Subject: [PATCH 2/7] Adapt Winsoriser indicators to narwhals-returning check_X With add_indicators=True, transform() compared the capped output against check_X(X), which is now a narwhals frame, so the pandas path mixed pandas and narwhals objects (broadcast errors, wrong indicators). Compare against the user's native X instead; _transform() already validates it. Co-Authored-By: Claude Opus 5 --- feature_engine/outliers/winsorizer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index 1a4fa5194..304ab255f 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -27,7 +27,6 @@ ) from feature_engine._docstrings.methods import _fit_transform_docstring from feature_engine._docstrings.substitute import Substitution -from feature_engine.dataframe_checks import check_X from feature_engine.outliers.base_outlier import WinsorizerBase @@ -217,8 +216,9 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: X_out = super()._transform(X) else: - X_orig = check_X(X) - X_out = super()._transform(X_orig) + # X_out is validated and reordered by _transform(); the indicators + # compare it against the user's native X, variable by variable. + X_out = super()._transform(X) # Benchmarked at 10k-100k rows x 1-10 columns: pandas-native # comparison + concat is up to ~3x faster than the narwhals @@ -228,7 +228,7 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: is_pandas = nwd.is_pandas_dataframe(X_out) if is_pandas is True: pd = nw.from_native(X_out, eager_only=True).__native_namespace__() - X_orig_filtered = X_orig[self.variables_] + X_orig_filtered = X[self.variables_] X_out_filtered = X_out[self.variables_] if self.tail in ["left", "both"]: @@ -254,7 +254,7 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: ] X_out = pd.concat([X_out, X_both], axis=1) else: - nw_orig = nw.from_native(X_orig, eager_only=True) + nw_orig = nw.from_native(X, eager_only=True) nw_out = nw.from_native(X_out, eager_only=True) new_cols = [] From 7374109c7a1bf82cad3ecdae5ef263d1a9b1e4e8 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:37:56 +0200 Subject: [PATCH 3/7] Use shared backend test fixtures and helpers in Winsoriser tests Replace the file-local data dicts and _col/_cols/_shape/_drop_missing helpers with the shared test structure: make_df and data_normal_dist / data_na fixtures, isinstance(X, make_df) plus to_dict() checks, and pytest.raises(match=...). Co-Authored-By: Claude Opus 5 --- tests/test_outliers/test_winsorizer.py | 269 ++++++++++--------------- 1 file changed, 112 insertions(+), 157 deletions(-) diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index e0fe62346..5ce9bc803 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -1,22 +1,21 @@ import math import re -import narwhals as nw -import numpy as np -import pandas as pd -import polars as pl import pytest from feature_engine.outliers import Winsoriser, Winsorizer +from tests.backend_helpers import to_dict DEPRECATION_WARNING = ( "Winsorizer was deprecated in favour of Winsoriser in version 2.0.0 and will " "be removed in version 2.1.0. To silence this warning, use Winsoriser instead." ) -# mirrors tests/conftest.py's df_vartypes fixture, minus the datetime "dob" -# column (values that don't round-trip identically between pandas and -# polars are irrelevant here, since Winsoriser only touches numerical vars) +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." +) + VARTYPES = { "Name": ["tom", "nick", "krish", "jack"], "City": ["London", "Manchester", "Liverpool", "Bristol"], @@ -24,56 +23,6 @@ "Marks": [0.9, 0.8, 0.7, 0.6], } -# mirrors tests/conftest.py's df_na fixture, minus "dob"; uses None (not -# np.nan) for missing values in string columns since polars' DataFrame -# constructor rejects a float NaN mixed into a string column -DATA_NA = { - "Name": ["tom", "nick", "krish", None, "peter", None, "fred", "sam"], - "City": [ - "London", - "Manchester", - None, - None, - "London", - "London", - "Bristol", - "Manchester", - ], - "Studies": [ - "Bachelor", - "Bachelor", - None, - None, - "Bachelor", - "PhD", - "None", - "Masters", - ], - "Age": [20, 21, 19, None, 23, 40, 41, 37], - "Marks": [0.9, 0.8, 0.7, None, 0.3, None, 0.8, 0.6], -} - - -def _col(X, col): - return nw.from_native(X, eager_only=True)[col].to_list() - - -def _cols(X): - return list(nw.from_native(X, eager_only=True).columns) - - -def _shape(X): - return nw.from_native(X, eager_only=True).shape - - -def _drop_missing(values): - # pandas yields float('nan') for a missing numeric value; polars yields - # None. Filter both so tests can assert on the same non-missing values - # regardless of backend. - return [ - v for v in values if v is not None and not (isinstance(v, float) and v != v) - ] - @pytest.fixture( params=[Winsoriser, Winsorizer], @@ -95,14 +44,14 @@ def test_winsorizer_raises_future_warning(): Winsorizer() -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_gaussian_capping_right_tail_with_fold_1(make_df, transformer_class): +def test_gaussian_capping_right_tail_with_fold_1( + make_df, data_normal_dist, transformer_class +): # test case 1: mean and std, right tail - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="gaussian", tail="right", fold=1 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test init params assert transformer.capping_method == "gaussian" @@ -113,117 +62,123 @@ def test_gaussian_capping_right_tail_with_fold_1(make_df, transformer_class): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform outputs - assert math.isclose(max(_col(X_out, "var")), 0.1067690260251065) + assert isinstance(X_out, make_df) + assert math.isclose(max(to_dict(X_out)["var"]), 0.1067690260251065) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_gaussian_capping_both_tails_with_fold_2(make_df, transformer_class): +def test_gaussian_capping_both_tails_with_fold_2( + make_df, data_normal_dist, transformer_class +): # test case 2: mean and std, both tails, different fold value - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="gaussian", tail="both", fold=2 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.2075572504967645) assert math.isclose(transformer.left_tail_caps_["var"], -0.1955956473898675) # test transform output - values = _col(X_out, "var") + assert isinstance(X_out, make_df) + values = to_dict(X_out)["var"] assert math.isclose(max(values), 0.2075572504967645) assert math.isclose(min(values), -0.1955956473898675) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_iqr_capping_both_tails_with_fold_1(make_df, transformer_class): +def test_iqr_capping_both_tails_with_fold_1( + make_df, data_normal_dist, transformer_class +): # test case 3: IQR, both tails, fold 1 - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="iqr", tail="both", fold=1 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.21180113880445128) assert math.isclose(transformer.left_tail_caps_["var"], -0.20247907173293223) # test transform output - values = _col(X_out, "var") + assert isinstance(X_out, make_df) + values = to_dict(X_out)["var"] assert math.isclose(max(values), 0.21180113880445128) assert math.isclose(min(values), -0.20247907173293223) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_iqr_capping_left_tail_with_fold_2(make_df, transformer_class): +def test_iqr_capping_left_tail_with_fold_2( + make_df, data_normal_dist, transformer_class +): # test case 4: IQR, left tail, fold 0.8 - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="iqr", tail="left", fold=0.8 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert transformer.right_tail_caps_ == {} assert math.isclose(transformer.left_tail_caps_["var"], -0.17486039103044) # test transform output - assert math.isclose(min(_col(X_out, "var")), -0.17486039103044) + assert isinstance(X_out, make_df) + assert math.isclose(min(to_dict(X_out)["var"]), -0.17486039103044) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_quantile_capping_both_tails_with_fold_10_percent(make_df, transformer_class): +def test_quantile_capping_both_tails_with_fold_10_percent( + make_df, data_normal_dist, transformer_class +): # test case 5: quantiles, both tails, fold 10% - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="quantiles", tail="both", fold=0.1 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.14712481122898166) assert math.isclose(transformer.left_tail_caps_["var"], -0.12366227743232801) # test transform output - values = _col(X_out, "var") + assert isinstance(X_out, make_df) + values = to_dict(X_out)["var"] assert math.isclose(max(values), 0.14712481122898166) assert math.isclose(min(values), -0.12366227743232801) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_quantile_capping_right_tail_with_fold_15_percent(make_df, transformer_class): +def test_quantile_capping_right_tail_with_fold_15_percent( + make_df, data_normal_dist, transformer_class +): # test case 6: quantiles, right tail, fold 15% - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="quantiles", tail="right", fold=0.15 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.11823196128033647) assert transformer.left_tail_caps_ == {} # test transform output - assert math.isclose(max(_col(X_out, "var")), 0.11823196128033647) + assert isinstance(X_out, make_df) + assert math.isclose(max(to_dict(X_out)["var"]), 0.11823196128033647) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) @pytest.mark.parametrize( "strings,expected", [("gaussian", 3), ("iqr", 1.5), ("mad", 3.29), ("quantiles", 0.05)], ) -def test_auto_fold_default_value(make_df, strings, expected, transformer_class): - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) +def test_auto_fold_default_value( + make_df, data_normal_dist, strings, expected, transformer_class +): transformer = make_transformer( transformer_class, capping_method=strings, fold="auto" ) - transformer.fit(X) + transformer.fit(make_df(data_normal_dist)) assert transformer.fold_ == expected -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_mad_capping_right_tail_with_fold_1(make_df, transformer_class): +def test_mad_capping_right_tail_with_fold_1( + make_df, data_normal_dist, transformer_class +): # test case: median and mad, right tail - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="mad", tail="right", fold=1 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test init params assert transformer.capping_method == "mad" @@ -234,31 +189,32 @@ def test_mad_capping_right_tail_with_fold_1(make_df, transformer_class): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform outputs - assert math.isclose(max(_col(X_out, "var")), 0.10995521088494983) + assert isinstance(X_out, make_df) + assert math.isclose(max(to_dict(X_out)["var"]), 0.10995521088494983) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_mad_capping_both_tails_with_fold_2(make_df, transformer_class): +def test_mad_capping_both_tails_with_fold_2( + make_df, data_normal_dist, transformer_class +): # test case: mad, both tails, different fold value - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) transformer = make_transformer( transformer_class, capping_method="mad", tail="both", fold=2 ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_normal_dist)) # test fit params assert math.isclose(transformer.right_tail_caps_["var"], 0.21050080982609987) assert math.isclose(transformer.left_tail_caps_["var"], -0.1916815859385002) # test transform output - values = _col(X_out, "var") + assert isinstance(X_out, make_df) + values = to_dict(X_out)["var"] assert math.isclose(max(values), 0.21050080982609987) assert math.isclose(min(values), -0.1916815859385002) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_indicators_are_added(make_df, transformer_class): - X = make_df({"var": np.random.RandomState(0).normal(0, 0.1, 100)}) - n_cols = _shape(X)[1] +def test_indicators_are_added(make_df, data_normal_dist, transformer_class): + X = make_df(data_normal_dist) + n_cols = X.shape[1] transformer = make_transformer( transformer_class, @@ -268,9 +224,11 @@ def test_indicators_are_added(make_df, transformer_class): add_indicators=True, ) X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == 3 * n_cols - for col in _cols(X_out)[n_cols:]: - assert sum(_col(X_out, col)) > 0 + assert isinstance(X_out, make_df) + assert X_out.shape[1] == 3 * n_cols + result = to_dict(X_out) + for col in list(X_out.columns)[n_cols:]: + assert sum(result[col]) > 0 transformer = make_transformer( transformer_class, @@ -280,9 +238,11 @@ def test_indicators_are_added(make_df, transformer_class): add_indicators=True, ) X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == 2 * n_cols - for col in _cols(X_out)[n_cols:]: - assert sum(_col(X_out, col)) > 0 + assert isinstance(X_out, make_df) + assert X_out.shape[1] == 2 * n_cols + result = to_dict(X_out) + for col in list(X_out.columns)[n_cols:]: + assert sum(result[col]) > 0 transformer = make_transformer( transformer_class, @@ -292,15 +252,16 @@ def test_indicators_are_added(make_df, transformer_class): add_indicators=True, ) X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == 2 * n_cols - for col in _cols(X_out)[n_cols:]: - assert sum(_col(X_out, col)) > 0 + assert isinstance(X_out, make_df) + assert X_out.shape[1] == 2 * n_cols + result = to_dict(X_out) + for col in list(X_out.columns)[n_cols:]: + assert sum(result[col]) > 0 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_indicators_filter_variables(make_df, transformer_class): X = make_df(VARTYPES) - n_cols = _shape(X)[1] + n_cols = X.shape[1] transformer = make_transformer( transformer_class, @@ -311,20 +272,20 @@ def test_indicators_filter_variables(make_df, transformer_class): add_indicators=True, ) X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == n_cols + 4 + assert isinstance(X_out, make_df) + assert X_out.shape[1] == n_cols + 4 transformer.set_params(tail="left") X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == n_cols + 2 + assert X_out.shape[1] == n_cols + 2 transformer.set_params(tail="right") X_out = transformer.fit_transform(X) - assert _shape(X_out)[1] == n_cols + 2 + assert X_out.shape[1] == n_cols + 2 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_indicators_are_correct(make_df, transformer_class): - X = make_df({"col": np.arange(100).astype(np.float64)}) + X = make_df({"col": [float(i) for i in range(100)]}) expected_left = [1.0] * 10 + [0.0] * 90 expected_right = [0.0] * 90 + [1.0] * 10 @@ -336,23 +297,23 @@ def test_indicators_are_correct(make_df, transformer_class): add_indicators=True, ) X_out = transformer.fit_transform(X) - assert _col(X_out, "col_left") == expected_left + assert isinstance(X_out, make_df) + assert to_dict(X_out)["col_left"] == expected_left transformer.set_params(tail="right") X_out = transformer.fit_transform(X) - assert _col(X_out, "col_right") == expected_right + assert to_dict(X_out)["col_right"] == expected_right transformer.set_params(tail="both") X_out = transformer.fit_transform(X) - assert _col(X_out, "col_left") == expected_left - assert _col(X_out, "col_right") == expected_right - assert _cols(X_out) == ["col", "col_left", "col_right"] + result = to_dict(X_out) + assert result["col_left"] == expected_left + assert result["col_right"] == expected_right + assert list(X_out.columns) == ["col", "col_left", "col_right"] -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_transformer_ignores_na_in_df(make_df, transformer_class): +def test_transformer_ignores_na_in_df(make_df, data_na, transformer_class): # test case: dataset contains na and transformer is asked to ignore them - X = make_df(DATA_NA) transformer = make_transformer( transformer_class, capping_method="gaussian", @@ -361,7 +322,7 @@ def test_transformer_ignores_na_in_df(make_df, transformer_class): variables=["Age", "Marks"], missing_values="ignore", ) - X_out = transformer.fit_transform(X) + X_out = transformer.fit_transform(make_df(data_na)) # test fit params assert math.isclose(transformer.right_tail_caps_["Age"], 38.04494616731882) @@ -369,8 +330,10 @@ def test_transformer_ignores_na_in_df(make_df, transformer_class): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 5 # test transform output - age = _drop_missing(_col(X_out, "Age")) - marks = _drop_missing(_col(X_out, "Marks")) + assert isinstance(X_out, make_df) + result = to_dict(X_out) + age = [v for v in result["Age"] if v is not None] + marks = [v for v in result["Marks"] if v is not None] assert math.isclose(max(age), 38.04494616731882) assert math.isclose(max(marks), 0.8784116651786605) @@ -411,32 +374,27 @@ def test_error_if_add_incators_not_permitted(transformer_class): make_transformer(transformer_class, add_indicators=[True]) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_fit_raises_error_if_na_in_inut_df(make_df, transformer_class): +def test_fit_raises_error_if_na_in_inut_df(make_df, data_na, transformer_class): # test case: when dataset contains na, fit method - X = make_df(DATA_NA) - with pytest.raises(ValueError): - transformer = make_transformer(transformer_class) - transformer.fit(X) + transformer = make_transformer(transformer_class) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.fit(make_df(data_na)) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_transform_raises_error_if_na_in_input_df(make_df, transformer_class): +def test_transform_raises_error_if_na_in_input_df( + make_df, data_na, transformer_class +): # test case: when dataset contains na, transform method - X_fit = make_df(VARTYPES) - X_na = make_df( - {k: DATA_NA[k] for k in ["Name", "City", "Age", "Marks"]} - ) - with pytest.raises(ValueError): - transformer = make_transformer(transformer_class) - transformer.fit(X_fit) + X_na = make_df({k: data_na[k] for k in ["Name", "City", "Age", "Marks"]}) + transformer = make_transformer(transformer_class) + transformer.fit(make_df(VARTYPES)) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): transformer.transform(X_na) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_get_feature_names_out(make_df, transformer_class): - X = make_df(DATA_NA) - original_features = _cols(X) +def test_get_feature_names_out(make_df, data_na, transformer_class): + X = make_df(data_na) + original_features = list(data_na) input_features = ["Age", "Marks"] # when indicators is false, we've got the generic check. @@ -478,11 +436,8 @@ def test_get_feature_names_out(make_df, transformer_class): assert tr.get_feature_names_out(original_features) == original_features + out -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) -def test_low_variation(make_df, transformer_class): - X = make_df( - {"var": (np.random.RandomState(0).normal(0, 0.1, 100) // 10).tolist()} - ) +def test_low_variation(make_df, data_normal_dist, transformer_class): + X = make_df({"var": [v // 10 for v in data_normal_dist["var"]]}) transformer = make_transformer(transformer_class, capping_method="mad") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="have low variation for method 'mad'"): transformer.fit(X) From e6164af04844740759f8f0420123aabd910d7ca7 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 12:02:03 +0200 Subject: [PATCH 4/7] Use frame_to_dict after the shared helper rename in #1045 Co-Authored-By: Claude Opus 5 --- tests/test_outliers/test_winsorizer.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index 5ce9bc803..8b47bf38f 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -4,7 +4,7 @@ import pytest from feature_engine.outliers import Winsoriser, Winsorizer -from tests.backend_helpers import to_dict +from tests.backend_helpers import frame_to_dict DEPRECATION_WARNING = ( "Winsorizer was deprecated in favour of Winsoriser in version 2.0.0 and will " @@ -63,7 +63,7 @@ def test_gaussian_capping_right_tail_with_fold_1( assert transformer.n_features_in_ == 1 # test transform outputs assert isinstance(X_out, make_df) - assert math.isclose(max(to_dict(X_out)["var"]), 0.1067690260251065) + assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.1067690260251065) def test_gaussian_capping_both_tails_with_fold_2( @@ -80,7 +80,7 @@ def test_gaussian_capping_both_tails_with_fold_2( assert math.isclose(transformer.left_tail_caps_["var"], -0.1955956473898675) # test transform output assert isinstance(X_out, make_df) - values = to_dict(X_out)["var"] + values = frame_to_dict(X_out)["var"] assert math.isclose(max(values), 0.2075572504967645) assert math.isclose(min(values), -0.1955956473898675) @@ -99,7 +99,7 @@ def test_iqr_capping_both_tails_with_fold_1( assert math.isclose(transformer.left_tail_caps_["var"], -0.20247907173293223) # test transform output assert isinstance(X_out, make_df) - values = to_dict(X_out)["var"] + values = frame_to_dict(X_out)["var"] assert math.isclose(max(values), 0.21180113880445128) assert math.isclose(min(values), -0.20247907173293223) @@ -118,7 +118,7 @@ def test_iqr_capping_left_tail_with_fold_2( assert math.isclose(transformer.left_tail_caps_["var"], -0.17486039103044) # test transform output assert isinstance(X_out, make_df) - assert math.isclose(min(to_dict(X_out)["var"]), -0.17486039103044) + assert math.isclose(min(frame_to_dict(X_out)["var"]), -0.17486039103044) def test_quantile_capping_both_tails_with_fold_10_percent( @@ -135,7 +135,7 @@ def test_quantile_capping_both_tails_with_fold_10_percent( assert math.isclose(transformer.left_tail_caps_["var"], -0.12366227743232801) # test transform output assert isinstance(X_out, make_df) - values = to_dict(X_out)["var"] + values = frame_to_dict(X_out)["var"] assert math.isclose(max(values), 0.14712481122898166) assert math.isclose(min(values), -0.12366227743232801) @@ -154,7 +154,7 @@ def test_quantile_capping_right_tail_with_fold_15_percent( assert transformer.left_tail_caps_ == {} # test transform output assert isinstance(X_out, make_df) - assert math.isclose(max(to_dict(X_out)["var"]), 0.11823196128033647) + assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.11823196128033647) @pytest.mark.parametrize( @@ -190,7 +190,7 @@ def test_mad_capping_right_tail_with_fold_1( assert transformer.n_features_in_ == 1 # test transform outputs assert isinstance(X_out, make_df) - assert math.isclose(max(to_dict(X_out)["var"]), 0.10995521088494983) + assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.10995521088494983) def test_mad_capping_both_tails_with_fold_2( @@ -207,7 +207,7 @@ def test_mad_capping_both_tails_with_fold_2( assert math.isclose(transformer.left_tail_caps_["var"], -0.1916815859385002) # test transform output assert isinstance(X_out, make_df) - values = to_dict(X_out)["var"] + values = frame_to_dict(X_out)["var"] assert math.isclose(max(values), 0.21050080982609987) assert math.isclose(min(values), -0.1916815859385002) @@ -226,7 +226,7 @@ def test_indicators_are_added(make_df, data_normal_dist, transformer_class): X_out = transformer.fit_transform(X) assert isinstance(X_out, make_df) assert X_out.shape[1] == 3 * n_cols - result = to_dict(X_out) + result = frame_to_dict(X_out) for col in list(X_out.columns)[n_cols:]: assert sum(result[col]) > 0 @@ -240,7 +240,7 @@ def test_indicators_are_added(make_df, data_normal_dist, transformer_class): X_out = transformer.fit_transform(X) assert isinstance(X_out, make_df) assert X_out.shape[1] == 2 * n_cols - result = to_dict(X_out) + result = frame_to_dict(X_out) for col in list(X_out.columns)[n_cols:]: assert sum(result[col]) > 0 @@ -254,7 +254,7 @@ def test_indicators_are_added(make_df, data_normal_dist, transformer_class): X_out = transformer.fit_transform(X) assert isinstance(X_out, make_df) assert X_out.shape[1] == 2 * n_cols - result = to_dict(X_out) + result = frame_to_dict(X_out) for col in list(X_out.columns)[n_cols:]: assert sum(result[col]) > 0 @@ -298,15 +298,15 @@ def test_indicators_are_correct(make_df, transformer_class): ) X_out = transformer.fit_transform(X) assert isinstance(X_out, make_df) - assert to_dict(X_out)["col_left"] == expected_left + assert frame_to_dict(X_out)["col_left"] == expected_left transformer.set_params(tail="right") X_out = transformer.fit_transform(X) - assert to_dict(X_out)["col_right"] == expected_right + assert frame_to_dict(X_out)["col_right"] == expected_right transformer.set_params(tail="both") X_out = transformer.fit_transform(X) - result = to_dict(X_out) + result = frame_to_dict(X_out) assert result["col_left"] == expected_left assert result["col_right"] == expected_right assert list(X_out.columns) == ["col", "col_left", "col_right"] @@ -331,7 +331,7 @@ def test_transformer_ignores_na_in_df(make_df, data_na, transformer_class): assert transformer.n_features_in_ == 5 # test transform output assert isinstance(X_out, make_df) - result = to_dict(X_out) + result = frame_to_dict(X_out) age = [v for v in result["Age"] if v is not None] marks = [v for v in result["Marks"] if v is not None] assert math.isclose(max(age), 38.04494616731882) From f5b4a0c819157cb6504086dc5f1aae233f849d41 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Fri, 18 Sep 2026 14:17:50 +0200 Subject: [PATCH 5/7] Call is_pandas_dataframe in the condition instead of storing it in Winsorizer Co-Authored-By: Claude Opus 5 --- feature_engine/outliers/winsorizer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index 304ab255f..761e1fe73 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -225,8 +225,7 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: # with_columns equivalent on pandas input (the loss grows with # column count), so pandas keeps its own fast path here, same # split as MissingIndicator's indicator-building step. - is_pandas = nwd.is_pandas_dataframe(X_out) - if is_pandas is True: + if nwd.is_pandas_dataframe(X_out) is True: pd = nw.from_native(X_out, eager_only=True).__native_namespace__() X_orig_filtered = X[self.variables_] X_out_filtered = X_out[self.variables_] From 0de46b183da74d041e109ccdc558ca2994973b3c Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 19 Sep 2026 08:42:24 +0200 Subject: [PATCH 6/7] Document and test infinite caps for variables without variation Co-Authored-By: Claude Opus 5 --- docs/user_guide/outliers/Winsoriser.rst | 7 +++++++ tests/test_outliers/test_winsorizer.py | 25 ++++++++++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/user_guide/outliers/Winsoriser.rst b/docs/user_guide/outliers/Winsoriser.rst index cf6fbee2d..a374f8e07 100644 --- a/docs/user_guide/outliers/Winsoriser.rst +++ b/docs/user_guide/outliers/Winsoriser.rst @@ -67,6 +67,13 @@ Percentiles or quantiles The values used by default by :class:`Winsoriser()` are those suggested as optimal in statistical studies. +.. note:: + + If all or most of the values of a variable are the same, the method may return a + spread of 0 (for example, an IQR of 0 when over half of the values are 0). The + variable then has no outliers, so :class:`Winsoriser()` sets its limits to infinity + in `right_tail_caps_` and `left_tail_caps_`, and leaves it untouched. + The following image shows the four methods applied to a normal distribution. Their capping values are close together because, when the data is roughly symmetric and bell-shaped, the mean, median, standard deviation, IQR, and MAD all describe the same thing. diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index 8b47bf38f..e81eabdd2 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -1,6 +1,7 @@ import math import re +import numpy as np import pytest from feature_engine.outliers import Winsoriser, Winsorizer @@ -436,8 +437,22 @@ def test_get_feature_names_out(make_df, data_na, transformer_class): assert tr.get_feature_names_out(original_features) == original_features + out -def test_low_variation(make_df, data_normal_dist, transformer_class): - X = make_df({"var": [v // 10 for v in data_normal_dist["var"]]}) - transformer = make_transformer(transformer_class, capping_method="mad") - with pytest.raises(ValueError, match="have low variation for method 'mad'"): - transformer.fit(X) +def test_variables_without_variation_are_left_untouched( + make_df, data_normal_dist, transformer_class +): + data = { + "var": [v // 10 for v in data_normal_dist["var"]], + "other": data_normal_dist["var"], + } + transformer = make_transformer( + transformer_class, capping_method="mad", tail="both", add_indicators=True + ) + Xt = transformer.fit_transform(make_df(data)) + + assert transformer.right_tail_caps_["var"] == np.inf + assert transformer.left_tail_caps_["var"] == -np.inf + assert isinstance(Xt, make_df) + result = frame_to_dict(Xt) + assert result["var"] == data["var"] + assert result["var_left"] == [0.0] * len(data["var"]) + assert result["var_right"] == [0.0] * len(data["var"]) From f1ac44fa172ae4761a7e50e077948d65c1c5387b Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 19 Sep 2026 09:02:28 +0200 Subject: [PATCH 7/7] Align Winsoriser and its tests with the repo conventions Co-Authored-By: Claude Opus 5 --- feature_engine/outliers/winsorizer.py | 17 +- tests/test_outliers/test_winsorizer.py | 379 +++++++------------------ 2 files changed, 110 insertions(+), 286 deletions(-) diff --git a/feature_engine/outliers/winsorizer.py b/feature_engine/outliers/winsorizer.py index 761e1fe73..63454f404 100644 --- a/feature_engine/outliers/winsorizer.py +++ b/feature_engine/outliers/winsorizer.py @@ -186,7 +186,7 @@ def __init__( ) -> None: if not isinstance(add_indicators, bool): raise ValueError( - "add_indicators takes only booleans True and False" + "add_indicators takes only booleans True and False. " f"Got {add_indicators} instead." ) super().__init__( @@ -212,19 +212,10 @@ def transform(self, X: IntoDataFrame) -> IntoDataFrame: to 'n_features', otherwise, will have an additional indicator column per processed feature for each tail. """ - if not self.add_indicators: - X_out = super()._transform(X) + X_out = super()._transform(X) - else: - # X_out is validated and reordered by _transform(); the indicators - # compare it against the user's native X, variable by variable. - X_out = super()._transform(X) - - # Benchmarked at 10k-100k rows x 1-10 columns: pandas-native - # comparison + concat is up to ~3x faster than the narwhals - # with_columns equivalent on pandas input (the loss grows with - # column count), so pandas keeps its own fast path here, same - # split as MissingIndicator's indicator-building step. + if self.add_indicators is True: + # pandas is faster than narwhals. if nwd.is_pandas_dataframe(X_out) is True: pd = nw.from_native(X_out, eager_only=True).__native_namespace__() X_orig_filtered = X[self.variables_] diff --git a/tests/test_outliers/test_winsorizer.py b/tests/test_outliers/test_winsorizer.py index e81eabdd2..ad1a2308d 100644 --- a/tests/test_outliers/test_winsorizer.py +++ b/tests/test_outliers/test_winsorizer.py @@ -1,4 +1,3 @@ -import math import re import numpy as np @@ -40,249 +39,146 @@ def make_transformer(transformer_class, **kwargs): return transformer_class(**kwargs) +# init parameters +# the errors of the parameters from WinsorizerBase are tested in test_base_outlier.py def test_winsorizer_raises_future_warning(): with pytest.warns(FutureWarning, match=re.escape(DEPRECATION_WARNING)): Winsorizer() -def test_gaussian_capping_right_tail_with_fold_1( - make_df, data_normal_dist, transformer_class -): - # test case 1: mean and std, right tail - transformer = make_transformer( - transformer_class, capping_method="gaussian", tail="right", fold=1 - ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test init params - assert transformer.capping_method == "gaussian" - assert transformer.tail == "right" - assert transformer.fold == 1 - # test fit attr - assert math.isclose(transformer.right_tail_caps_["var"], 0.1067690260251065) - assert transformer.left_tail_caps_ == {} - assert transformer.n_features_in_ == 1 - # test transform outputs - assert isinstance(X_out, make_df) - assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.1067690260251065) - - -def test_gaussian_capping_both_tails_with_fold_2( - make_df, data_normal_dist, transformer_class -): - # test case 2: mean and std, both tails, different fold value - transformer = make_transformer( - transformer_class, capping_method="gaussian", tail="both", fold=2 - ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test fit params - assert math.isclose(transformer.right_tail_caps_["var"], 0.2075572504967645) - assert math.isclose(transformer.left_tail_caps_["var"], -0.1955956473898675) - # test transform output - assert isinstance(X_out, make_df) - values = frame_to_dict(X_out)["var"] - assert math.isclose(max(values), 0.2075572504967645) - assert math.isclose(min(values), -0.1955956473898675) - - -def test_iqr_capping_both_tails_with_fold_1( - make_df, data_normal_dist, transformer_class -): - # test case 3: IQR, both tails, fold 1 - transformer = make_transformer( - transformer_class, capping_method="iqr", tail="both", fold=1 +@pytest.mark.parametrize("add_indicators", [-1, 1, "True", None, (), [True]]) +def test_error_if_add_indicators_not_permitted(add_indicators, transformer_class): + msg = ( + "add_indicators takes only booleans True and False. " + f"Got {add_indicators} instead." ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test fit params - assert math.isclose(transformer.right_tail_caps_["var"], 0.21180113880445128) - assert math.isclose(transformer.left_tail_caps_["var"], -0.20247907173293223) - # test transform output - assert isinstance(X_out, make_df) - values = frame_to_dict(X_out)["var"] - assert math.isclose(max(values), 0.21180113880445128) - assert math.isclose(min(values), -0.20247907173293223) - - -def test_iqr_capping_left_tail_with_fold_2( - make_df, data_normal_dist, transformer_class -): - # test case 4: IQR, left tail, fold 0.8 - transformer = make_transformer( - transformer_class, capping_method="iqr", tail="left", fold=0.8 - ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test fit params - assert transformer.right_tail_caps_ == {} - assert math.isclose(transformer.left_tail_caps_["var"], -0.17486039103044) - # test transform output - assert isinstance(X_out, make_df) - assert math.isclose(min(frame_to_dict(X_out)["var"]), -0.17486039103044) + with pytest.raises(ValueError, match=re.escape(msg)): + make_transformer(transformer_class, add_indicators=add_indicators) -def test_quantile_capping_both_tails_with_fold_10_percent( - make_df, data_normal_dist, transformer_class +@pytest.mark.parametrize( + "capping_method, tail, fold, add_indicators, missing_values", + [ + ("gaussian", "right", "auto", False, "raise"), + ("iqr", "left", 2, True, "ignore"), + ("mad", "both", 1.5, False, "ignore"), + ("quantiles", "both", 0.1, True, "raise"), + ], +) +def test_init_param_assignment( + capping_method, tail, fold, add_indicators, missing_values, transformer_class ): - # test case 5: quantiles, both tails, fold 10% transformer = make_transformer( - transformer_class, capping_method="quantiles", tail="both", fold=0.1 + transformer_class, + capping_method=capping_method, + tail=tail, + fold=fold, + add_indicators=add_indicators, + missing_values=missing_values, ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test fit params - assert math.isclose(transformer.right_tail_caps_["var"], 0.14712481122898166) - assert math.isclose(transformer.left_tail_caps_["var"], -0.12366227743232801) - # test transform output - assert isinstance(X_out, make_df) - values = frame_to_dict(X_out)["var"] - assert math.isclose(max(values), 0.14712481122898166) - assert math.isclose(min(values), -0.12366227743232801) + assert transformer.capping_method == capping_method + assert transformer.tail == tail + assert transformer.fold == fold + assert transformer.add_indicators == add_indicators + assert transformer.missing_values == missing_values -def test_quantile_capping_right_tail_with_fold_15_percent( - make_df, data_normal_dist, transformer_class +# fit and transform +@pytest.mark.parametrize( + "capping_method, tail, fold, right, left", + [ + ("gaussian", "right", 1, 0.1067690260251065, None), + ("gaussian", "both", 2, 0.2075572504967645, -0.1955956473898675), + ("iqr", "both", 1, 0.21180113880445128, -0.20247907173293223), + ("iqr", "left", 0.8, None, -0.17486039103044), + ("quantiles", "both", 0.1, 0.14712481122898166, -0.12366227743232801), + ("quantiles", "right", 0.15, 0.11823196128033647, None), + ("mad", "right", 1, 0.10995521088494983, None), + ("mad", "both", 2, 0.21050080982609987, -0.1916815859385002), + ], +) +def test_capping( + make_df, + data_normal_dist, + transformer_class, + capping_method, + tail, + fold, + right, + left, ): - # test case 6: quantiles, right tail, fold 15% transformer = make_transformer( - transformer_class, capping_method="quantiles", tail="right", fold=0.15 + transformer_class, capping_method=capping_method, tail=tail, fold=fold ) X_out = transformer.fit_transform(make_df(data_normal_dist)) - # test fit params - assert math.isclose(transformer.right_tail_caps_["var"], 0.11823196128033647) - assert transformer.left_tail_caps_ == {} - # test transform output + upper = np.inf if right is None else right + lower = -np.inf if left is None else left + expected = [min(max(v, lower), upper) for v in data_normal_dist["var"]] + + if right is None: + assert transformer.right_tail_caps_ == {} + else: + assert transformer.right_tail_caps_ == {"var": pytest.approx(right)} + if left is None: + assert transformer.left_tail_caps_ == {} + else: + assert transformer.left_tail_caps_ == {"var": pytest.approx(left)} + assert transformer.n_features_in_ == 1 assert isinstance(X_out, make_df) - assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.11823196128033647) + assert frame_to_dict(X_out) == {"var": pytest.approx(expected)} @pytest.mark.parametrize( - "strings,expected", + "capping_method, expected", [("gaussian", 3), ("iqr", 1.5), ("mad", 3.29), ("quantiles", 0.05)], ) def test_auto_fold_default_value( - make_df, data_normal_dist, strings, expected, transformer_class + make_df, data_normal_dist, capping_method, expected, transformer_class ): transformer = make_transformer( - transformer_class, capping_method=strings, fold="auto" + transformer_class, capping_method=capping_method, fold="auto" ) transformer.fit(make_df(data_normal_dist)) assert transformer.fold_ == expected -def test_mad_capping_right_tail_with_fold_1( - make_df, data_normal_dist, transformer_class -): - # test case: median and mad, right tail - transformer = make_transformer( - transformer_class, capping_method="mad", tail="right", fold=1 - ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test init params - assert transformer.capping_method == "mad" - assert transformer.tail == "right" - assert transformer.fold == 1 - # test fit attr - assert math.isclose(transformer.right_tail_caps_["var"], 0.10995521088494983) - assert transformer.left_tail_caps_ == {} - assert transformer.n_features_in_ == 1 - # test transform outputs - assert isinstance(X_out, make_df) - assert math.isclose(max(frame_to_dict(X_out)["var"]), 0.10995521088494983) - - -def test_mad_capping_both_tails_with_fold_2( - make_df, data_normal_dist, transformer_class +@pytest.mark.parametrize("tail, n_indicators", [("both", 2), ("left", 1), ("right", 1)]) +def test_indicators_are_added( + make_df, data_normal_dist, transformer_class, tail, n_indicators ): - # test case: mad, both tails, different fold value - transformer = make_transformer( - transformer_class, capping_method="mad", tail="both", fold=2 - ) - X_out = transformer.fit_transform(make_df(data_normal_dist)) - - # test fit params - assert math.isclose(transformer.right_tail_caps_["var"], 0.21050080982609987) - assert math.isclose(transformer.left_tail_caps_["var"], -0.1916815859385002) - # test transform output - assert isinstance(X_out, make_df) - values = frame_to_dict(X_out)["var"] - assert math.isclose(max(values), 0.21050080982609987) - assert math.isclose(min(values), -0.1916815859385002) - - -def test_indicators_are_added(make_df, data_normal_dist, transformer_class): X = make_df(data_normal_dist) - n_cols = X.shape[1] - transformer = make_transformer( transformer_class, - tail="both", + tail=tail, capping_method="quantiles", fold=0.1, add_indicators=True, ) X_out = transformer.fit_transform(X) - assert isinstance(X_out, make_df) - assert X_out.shape[1] == 3 * n_cols - result = frame_to_dict(X_out) - for col in list(X_out.columns)[n_cols:]: - assert sum(result[col]) > 0 - transformer = make_transformer( - transformer_class, - tail="left", - capping_method="quantiles", - fold=0.1, - add_indicators=True, - ) - X_out = transformer.fit_transform(X) - assert isinstance(X_out, make_df) - assert X_out.shape[1] == 2 * n_cols - result = frame_to_dict(X_out) - for col in list(X_out.columns)[n_cols:]: - assert sum(result[col]) > 0 - - transformer = make_transformer( - transformer_class, - tail="right", - capping_method="quantiles", - fold=0.1, - add_indicators=True, - ) - X_out = transformer.fit_transform(X) assert isinstance(X_out, make_df) - assert X_out.shape[1] == 2 * n_cols + assert X_out.shape[1] == 1 + n_indicators result = frame_to_dict(X_out) - for col in list(X_out.columns)[n_cols:]: + for col in list(X_out.columns)[1:]: assert sum(result[col]) > 0 -def test_indicators_filter_variables(make_df, transformer_class): +@pytest.mark.parametrize("tail, n_indicators", [("both", 4), ("left", 2), ("right", 2)]) +def test_indicators_filter_variables(make_df, transformer_class, tail, n_indicators): X = make_df(VARTYPES) - n_cols = X.shape[1] - transformer = make_transformer( transformer_class, variables=["Age", "Marks"], - tail="both", + tail=tail, capping_method="quantiles", fold=0.1, add_indicators=True, ) X_out = transformer.fit_transform(X) - assert isinstance(X_out, make_df) - assert X_out.shape[1] == n_cols + 4 - transformer.set_params(tail="left") - X_out = transformer.fit_transform(X) - assert X_out.shape[1] == n_cols + 2 - - transformer.set_params(tail="right") - X_out = transformer.fit_transform(X) - assert X_out.shape[1] == n_cols + 2 + assert isinstance(X_out, make_df) + assert X_out.shape[1] == len(VARTYPES) + n_indicators def test_indicators_are_correct(make_df, transformer_class): @@ -314,7 +210,6 @@ def test_indicators_are_correct(make_df, transformer_class): def test_transformer_ignores_na_in_df(make_df, data_na, transformer_class): - # test case: dataset contains na and transformer is asked to ignore them transformer = make_transformer( transformer_class, capping_method="gaussian", @@ -325,58 +220,20 @@ def test_transformer_ignores_na_in_df(make_df, data_na, transformer_class): ) X_out = transformer.fit_transform(make_df(data_na)) - # test fit params - assert math.isclose(transformer.right_tail_caps_["Age"], 38.04494616731882) - assert math.isclose(transformer.right_tail_caps_["Marks"], 0.8784116651786605) + assert transformer.right_tail_caps_ == { + "Age": pytest.approx(38.04494616731882), + "Marks": pytest.approx(0.8784116651786605), + } assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 5 - # test transform output assert isinstance(X_out, make_df) result = frame_to_dict(X_out) - age = [v for v in result["Age"] if v is not None] - marks = [v for v in result["Marks"] if v is not None] - assert math.isclose(max(age), 38.04494616731882) - assert math.isclose(max(marks), 0.8784116651786605) - - -def test_error_if_capping_method_not_permitted(transformer_class): - with pytest.raises(ValueError): - make_transformer(transformer_class, capping_method="other") - + for var, cap in [("Age", 38.04494616731882), ("Marks", 0.8784116651786605)]: + expected = [None if v is None else min(v, cap) for v in data_na[var]] + assert result[var] == pytest.approx(expected) -def test_error_if_tail_value_not_permitted(transformer_class): - with pytest.raises(ValueError): - make_transformer(transformer_class, tail="other") - -def test_error_if_missing_values_not_permited(transformer_class): - with pytest.raises(ValueError): - make_transformer(transformer_class, missing_values="other") - - -def test_error_if_fold_value_not_permitted(transformer_class): - with pytest.raises(ValueError): - make_transformer(transformer_class, fold=-1) - - -def test_error_if_capping_method_quantiles_and_fold_value_not_permitted( - transformer_class, -): - with pytest.raises(ValueError): - make_transformer(transformer_class, capping_method="quantiles", fold=0.3) - - -def test_error_if_add_incators_not_permitted(transformer_class): - with pytest.raises(ValueError): - make_transformer(transformer_class, add_indicators=-1) - with pytest.raises(ValueError): - make_transformer(transformer_class, add_indicators=()) - with pytest.raises(ValueError): - make_transformer(transformer_class, add_indicators=[True]) - - -def test_fit_raises_error_if_na_in_inut_df(make_df, data_na, transformer_class): - # test case: when dataset contains na, fit method +def test_fit_raises_error_if_na_in_input_df(make_df, data_na, transformer_class): transformer = make_transformer(transformer_class) with pytest.raises(ValueError, match=re.escape(MSG_NA)): transformer.fit(make_df(data_na)) @@ -385,7 +242,6 @@ def test_fit_raises_error_if_na_in_inut_df(make_df, data_na, transformer_class): def test_transform_raises_error_if_na_in_input_df( make_df, data_na, transformer_class ): - # test case: when dataset contains na, transform method X_na = make_df({k: data_na[k] for k in ["Name", "City", "Age", "Marks"]}) transformer = make_transformer(transformer_class) transformer.fit(make_df(VARTYPES)) @@ -393,48 +249,25 @@ def test_transform_raises_error_if_na_in_input_df( transformer.transform(X_na) -def test_get_feature_names_out(make_df, data_na, transformer_class): - X = make_df(data_na) +# without indicators, the feature names are covered by the generic checks +@pytest.mark.parametrize( + "tail, indicators", + [ + ("left", ["Age_left", "Marks_left"]), + ("right", ["Age_right", "Marks_right"]), + ("both", ["Age_left", "Age_right", "Marks_left", "Marks_right"]), + ], +) +def test_get_feature_names_out(make_df, data_na, transformer_class, tail, indicators): original_features = list(data_na) - input_features = ["Age", "Marks"] - - # when indicators is false, we've got the generic check. - # We need to test only when true tr = make_transformer( - transformer_class, - tail="left", - add_indicators=True, - missing_values="ignore", - ) - tr.fit(X) - - out = [f + "_left" for f in input_features] - assert tr.get_feature_names_out() == original_features + out - assert tr.get_feature_names_out(original_features) == original_features + out - - tr = make_transformer( - transformer_class, - tail="right", - add_indicators=True, - missing_values="ignore", - ) - tr.fit(X) - - out = [f + "_right" for f in input_features] - assert tr.get_feature_names_out() == original_features + out - assert tr.get_feature_names_out(original_features) == original_features + out - - tr = make_transformer( - transformer_class, - tail="both", - add_indicators=True, - missing_values="ignore", + transformer_class, tail=tail, add_indicators=True, missing_values="ignore" ) - tr.fit(X) + tr.fit(make_df(data_na)) - out = ["Age_left", "Age_right", "Marks_left", "Marks_right"] - assert tr.get_feature_names_out() == original_features + out - assert tr.get_feature_names_out(original_features) == original_features + out + expected = original_features + indicators + assert tr.get_feature_names_out() == expected + assert tr.get_feature_names_out(original_features) == expected def test_variables_without_variation_are_left_untouched(