From c53cf2ad6be1eda6eb1a5d1c3191f9ef4871e648 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Wed, 26 Aug 2026 00:59:24 +0200 Subject: [PATCH 1/6] Migrate ArbitraryOutlierCapper to narwhals, add polars support fit() only builds dicts from user input and validates variables/dtypes via check_numerical_variables (already narwhals-generic) - no numeric computation, so nothing to branch on there. The only pandas-specific lines were the feature_names_in_ assignment (X.columns.to_list(), a pandas-Index method), replaced with the same is_pandas-guarded pattern WinsorizerBase.fit() already uses (list(X.columns) for pandas, nw.from_native(X).columns - already list[str] - otherwise). transform() was already dataframe-agnostic via BaseOutlier._transform(); only its type hints changed (pd.DataFrame -> IntoDataFrame). Benchmarked fit+transform end-to-end at 10k/50k/100k rows x 1/2/10 columns: pandas-native (pre-migration) vs the migrated code on pandas were within noise of each other (~0.9-1.1x), and polars ran 2-4x faster than pandas on both. No pandas/polars branch needed - merged single path, consistent with the is_pandas-only-for-.columns precedent already set in WinsorizerBase. Confirmed the module needs zero pandas: reloaded artbitrary.py in isolation with sys.modules["pandas"] = None (simulating an uninstalled pandas) and ran fit/transform end-to-end on a polars frame - works, and int64 stays int64 for a same-dtype capping dict (the class docstring's own x1 example). Found, while doing so, a real dtype-preservation bug in the already- merged BaseOutlier._transform() (base_outlier.py, commit 71bf7cf on this branch's base) that predates this migration and is not introduced here: when a capping-dict spans columns of different dtypes that land in the same bound-group (e.g. max_capping_dict={"age": 50, "fare": 200} with age int64 and fare float64 - both "right_only"), the group's columns are stacked into one 2D array via to_numpy() before np.clip, which forces a common dtype and upcasts age to float64. The pre- narwhals code (verified against 71bf7cf^) clipped each column independently (X[feature] = X[feature].clip(...)), so int columns never picked up a neighboring float column's dtype. Confirmed this reproduces identically on both pandas and polars (same merged code path) and is untouched by this commit - it lives in base_outlier.py, shared with Winsoriser/OutlierTrimmer, out of this file's scope. Flagged separately rather than fixed here. Rewrote test_arbitrary_capper.py to one parametrized test per behavior over pd.DataFrame/pl.DataFrame (previously pandas-only), using nw.from_native(...).to_dict(as_series=False) for backend-agnostic assertions in place of pd.testing.assert_frame_equal, following the same pattern used for ReciprocalTransformer/ArcsinTransformer. Added a verified "With polars" section to the docs (float dtypes throughout, to sidestep the dtype-upcast issue above rather than put an unexplained surprise in a user-facing example); left the pre-existing pandas Titanic walkthrough untouched - no network access in this environment to re-verify the fetch_openml/CSV-backed output. Verified: tests/test_outliers full suite - 88 passed (up from 83, all 5 new instances are the added polars parametrizations), same 3 pre-existing check_estimator failures as the pre-migration baseline (numpy-array input, unrelated to this change). flake8 and mypy clean. sphinx -W build clean (only the pre-existing linkcode_resolve warning). Co-Authored-By: Claude Sonnet 5 --- .../outliers/ArbitraryOutlierCapper.rst | 38 ++++++ feature_engine/outliers/artbitrary.py | 24 ++-- tests/test_outliers/test_arbitrary_capper.py | 123 ++++++++++-------- 3 files changed, 122 insertions(+), 63 deletions(-) diff --git a/docs/user_guide/outliers/ArbitraryOutlierCapper.rst b/docs/user_guide/outliers/ArbitraryOutlierCapper.rst index 70153b25b..ce916e9c4 100644 --- a/docs/user_guide/outliers/ArbitraryOutlierCapper.rst +++ b/docs/user_guide/outliers/ArbitraryOutlierCapper.rst @@ -96,6 +96,44 @@ values: dtype: float64 +With polars +----------- + +:class:`ArbitraryOutlierCapper()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.outliers import ArbitraryOutlierCapper + + df = pl.DataFrame({ + "age": [20.0, 21.0, 19.0, 45.0, 67.0, 18.0, 90.0, 34.0, 55.0, 23.0], + "fare": [7.5, 8.0, 71.3, 13.0, 30.5, 7.9, 512.3, 26.0, 15.5, 8.6], + }) + + capper = ArbitraryOutlierCapper( + max_capping_dict={"age": 50, "fare": 200}, + min_capping_dict=None, + ) + + capper.fit(df) + Xt = capper.transform(df) + + print(Xt.select(["age", "fare"]).max()) + +The resulting maximum values, capped at the values we entered in the dictionary: + +.. code:: text + + shape: (1, 2) + ┌──────┬───────┐ + │ age ┆ fare │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞══════╪═══════╡ + │ 50.0 ┆ 200.0 │ + └──────┴───────┘ + Additional resources -------------------- diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index 6088520da..d30b774b1 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -4,7 +4,9 @@ from typing import Optional -import pandas as pd +import narwhals as nw +import narwhals.dependencies as nwd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._check_init_parameters.check_input_dictionary import ( _check_numerical_dict, @@ -134,16 +136,16 @@ def __init__( self.min_capping_dict = min_capping_dict self.missing_values = missing_values - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn any parameter. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. - y: pandas Series, default=None + y: Series, default=None y is not needed in this transformer. You can pass y or None. """ X = check_X(X) @@ -176,23 +178,29 @@ def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): else: self.left_tail_caps_ = {} - self.feature_names_in_ = X.columns.to_list() + # pandas' .columns is an Index, not a list - list() is required there; + # narwhals' .columns is already list[str]. + is_pandas = nwd.is_pandas_dataframe(X) + if is_pandas is True: + self.feature_names_in_ = list(X.columns) + else: + self.feature_names_in_ = nw.from_native(X, eager_only=True).columns self.n_features_in_ = X.shape[1] return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Cap the variable values. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe of shape = [n_samples, n_features] + X_new: dataframe of shape = [n_samples, n_features] The dataframe with the capped variables. """ return super()._transform(X) diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index 5cba357b8..dae074295 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -1,22 +1,36 @@ +import narwhals as nw import numpy as np import pandas as pd +import polars as pl import pytest from feature_engine.outliers import ArbitraryOutlierCapper +DATA = {"var": list(np.random.RandomState(0).normal(0, 0.1, 20))} -def test_right_end_capping(df_normal_dist): - # test case 1: right end capping +DATA_NA = { + "Name": ["tom", "nick", "krish", "jack", "tom", "eric"], + "City": ["London", "Manchester", "Liverpool", "Bristol", "Manchester", "Liverpool"], + "Age": [20.0, 21.0, 19.0, 18.0, np.nan, 41.0], + "Marks": [0.9, 0.8, 0.7, 0.6, 0.5, 0.6], + "dob": pd.date_range("2020-02-24", periods=6, freq="min"), +} + + +def _to_dict(X): + return nw.from_native(X, eager_only=True).to_dict(as_series=False) + + +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_right_end_capping(make_df): + X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict={"var": 0.10727677848029868}, min_capping_dict=None ) - X = transformer.fit_transform(df_normal_dist) + Xt = transformer.fit_transform(X) # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] > 0.10727677848029868, 0.10727677848029868, df_transf["var"] - ) + expected = [min(v, 0.10727677848029868) for v in DATA["var"]] # test init params assert np.round(transformer.max_capping_dict["var"], 3) == np.round( @@ -31,27 +45,24 @@ def test_right_end_capping(df_normal_dist): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].max(), 3) <= np.round(0.10727677848029868, 3) - assert np.round(df_normal_dist["var"].max(), 3) > np.round(0.10727677848029868, 3) + result = _to_dict(Xt) + assert result["var"] == pytest.approx(expected) + assert max(result["var"]) <= 0.10727677848029868 + 1e-8 -def test_both_ends_capping(df_normal_dist): - # test case 2: both tails +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_both_ends_capping(make_df): + X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict={"var": 0.20857275540714884}, min_capping_dict={"var": -0.19661115230025186}, ) - X = transformer.fit_transform(df_normal_dist) + Xt = transformer.fit_transform(X) # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] > 0.20857275540714884, 0.20857275540714884, df_transf["var"] - ) - df_transf["var"] = np.where( - df_transf["var"] < -0.19661115230025186, -0.19661115230025186, df_transf["var"] - ) + expected = [ + min(max(v, -0.19661115230025186), 0.20857275540714884) for v in DATA["var"] + ] # test fit params assert np.round(transformer.right_tail_caps_["var"], 3) == np.round( @@ -61,25 +72,22 @@ def test_both_ends_capping(df_normal_dist): -0.19661115230025186, 3 ) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].max(), 3) <= np.round(0.20857275540714884, 3) - assert np.round(X["var"].min(), 3) >= np.round(-0.19661115230025186, 3) - assert np.round(df_normal_dist["var"].max(), 3) > np.round(0.20857275540714884, 3) - assert np.round(df_normal_dist["var"].min(), 3) < np.round(-0.19661115230025186, 3) + result = _to_dict(Xt) + assert result["var"] == pytest.approx(expected) + assert max(result["var"]) <= 0.20857275540714884 + 1e-8 + assert min(result["var"]) >= -0.19661115230025186 - 1e-8 -def test_left_tail_capping(df_normal_dist): - # test case 3: left tail +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_left_tail_capping(make_df): + X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict=None, min_capping_dict={"var": -0.17486039103044} ) - X = transformer.fit_transform(df_normal_dist) + Xt = transformer.fit_transform(X) # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] < -0.17486039103044, -0.17486039103044, df_transf["var"] - ) + expected = [max(v, -0.17486039103044) for v in DATA["var"]] # test init param assert transformer.max_capping_dict is None @@ -92,30 +100,32 @@ def test_left_tail_capping(df_normal_dist): -0.17486039103044, 3 ) # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].min(), 3) >= np.round(-0.17486039103044, 3) - assert np.round(df_normal_dist["var"].min(), 3) < np.round(-0.17486039103044, 3) + result = _to_dict(Xt) + assert result["var"] == pytest.approx(expected) + assert min(result["var"]) >= -0.17486039103044 - 1e-8 -def test_ignores_na_in_input_df(df_na): - # test case 4: dataset contains na and transformer is asked to ignore them +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_ignores_na_in_input_df(make_df): + X = make_df(DATA_NA) transformer = ArbitraryOutlierCapper( max_capping_dict=None, min_capping_dict={"Age": 20}, missing_values="ignore" ) - X = transformer.fit_transform(df_na) + Xt = transformer.fit_transform(X) # expected output - df_transf = df_na.copy() - df_transf["Age"] = np.where(df_transf["Age"] < 20, 20, df_transf["Age"]) + expected = [ + v if np.isnan(v) else max(v, 20) for v in DATA_NA["Age"] + ] # test fit params assert transformer.max_capping_dict is None assert transformer.min_capping_dict == {"Age": 20} - assert transformer.n_features_in_ == 6 + assert transformer.n_features_in_ == 5 # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert X["Age"].min() >= 20 - assert df_na["Age"].min() < 20 + result = _to_dict(Xt) + assert result["Age"] == pytest.approx(expected, nan_ok=True) + assert np.nanmin(result["Age"]) >= 20 def test_error_if_max_capping_dict_wrong_input(): @@ -142,24 +152,29 @@ def test_error_if_missing_values_not_bool(): ArbitraryOutlierCapper(missing_values="other") -def test_fit_and_transform_raise_error_if_df_contains_na(df_normal_dist): - df_na = df_normal_dist.copy() - df_na.loc[1, "var"] = np.nan +@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) +def test_fit_and_transform_raise_error_if_df_contains_na(make_df): + X = make_df(DATA) + data_na = dict(DATA) + var_na = list(DATA["var"]) + var_na[1] = np.nan + data_na["var"] = var_na + X_na = make_df(data_na) - # test case 5: when dataset contains na, fit method + # test case: when dataset contains na, fit method with pytest.raises(ValueError): transformer = ArbitraryOutlierCapper( min_capping_dict={"var": -0.17486039103044} ) - transformer.fit(df_na) + transformer.fit(X_na) - # test case 6: when dataset contains na, transform method + # test case: when dataset contains na, transform method with pytest.raises(ValueError): transformer = ArbitraryOutlierCapper( min_capping_dict={"var": -0.17486039103044} ) - transformer.fit(df_normal_dist) - transformer.transform(df_na) + transformer.fit(X) + transformer.transform(X_na) @pytest.mark.parametrize( @@ -168,9 +183,7 @@ def test_fit_and_transform_raise_error_if_df_contains_na(df_normal_dist): ) def test_error_if_missing_values_wrong_type(missing_values): msg = "missing_values takes only values 'raise' or 'ignore'" - with pytest.raises(ValueError) as record: + with pytest.raises(ValueError, match=msg): ArbitraryOutlierCapper( min_capping_dict={"var": -0.17486039103044}, missing_values="missing_values" ) - # check that error message matches - assert str(record.value) == msg From 1f217fcfabc1c827ddfa88897afe942326de4a84 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:36:52 +0200 Subject: [PATCH 2/6] Adapt ArbitraryOutlierCapper to narwhals-returning check_X Bind the narwhals frame returned by check_X and set feature_names_in_ and n_features_in_ from it, instead of treating the check_X result as a native frame, mirroring the imputation and encoding modules. Co-Authored-By: Claude Opus 5 --- feature_engine/outliers/artbitrary.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index d30b774b1..ba8febf99 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -4,8 +4,6 @@ from typing import Optional -import narwhals as nw -import narwhals.dependencies as nwd from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._check_init_parameters.check_input_dictionary import ( @@ -148,7 +146,7 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): y: Series, default=None y is not needed in this transformer. You can pass y or None. """ - X = check_X(X) + nw_X = check_X(X) # find variables to be capped if self.min_capping_dict is None and self.max_capping_dict: @@ -178,14 +176,10 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): else: self.left_tail_caps_ = {} - # pandas' .columns is an Index, not a list - list() is required there; - # narwhals' .columns is already list[str]. - is_pandas = nwd.is_pandas_dataframe(X) - if is_pandas is True: - self.feature_names_in_ = list(X.columns) - else: - self.feature_names_in_ = nw.from_native(X, eager_only=True).columns - self.n_features_in_ = X.shape[1] + # list() normalises both a narwhals `.columns` (already a list) and a + # pandas Index to a plain list. + self.feature_names_in_ = list(nw_X.columns) + self.n_features_in_ = nw_X.shape[1] return self From 53c4717e80b109fdede0083b0123da1baf5b6d04 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Mon, 14 Sep 2026 22:36:53 +0200 Subject: [PATCH 3/6] Use shared backend test fixtures and helpers in ArbitraryOutlierCapper tests Replace the file-local _to_dict helper and parametrize decorators with the shared test structure: make_df fixture, isinstance(X, make_df) plus to_dict() checks, missing values written as None, and pytest.raises(match=re.escape(msg)). Co-Authored-By: Claude Opus 5 --- tests/test_outliers/test_arbitrary_capper.py | 94 +++++++++----------- 1 file changed, 40 insertions(+), 54 deletions(-) diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index dae074295..9231c6856 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -1,33 +1,33 @@ -import narwhals as nw +import datetime +import re + import numpy as np -import pandas as pd -import polars as pl import pytest from feature_engine.outliers import ArbitraryOutlierCapper +from tests.backend_helpers import to_dict -DATA = {"var": list(np.random.RandomState(0).normal(0, 0.1, 20))} +DATA = {"var": np.random.RandomState(0).normal(0, 0.1, 20).tolist()} DATA_NA = { "Name": ["tom", "nick", "krish", "jack", "tom", "eric"], "City": ["London", "Manchester", "Liverpool", "Bristol", "Manchester", "Liverpool"], - "Age": [20.0, 21.0, 19.0, 18.0, np.nan, 41.0], + "Age": [20.0, 21.0, 19.0, 18.0, None, 41.0], "Marks": [0.9, 0.8, 0.7, 0.6, 0.5, 0.6], - "dob": pd.date_range("2020-02-24", periods=6, freq="min"), + "dob": [datetime.datetime(2020, 2, 24, 0, i) for i in range(6)], } - -def _to_dict(X): - return nw.from_native(X, eager_only=True).to_dict(as_series=False) +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." +) -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_right_end_capping(make_df): - X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict={"var": 0.10727677848029868}, min_capping_dict=None ) - Xt = transformer.fit_transform(X) + Xt = transformer.fit_transform(make_df(DATA)) # expected output expected = [min(v, 0.10727677848029868) for v in DATA["var"]] @@ -45,19 +45,17 @@ def test_right_end_capping(make_df): assert transformer.left_tail_caps_ == {} assert transformer.n_features_in_ == 1 # test transform output - result = _to_dict(Xt) - assert result["var"] == pytest.approx(expected) - assert max(result["var"]) <= 0.10727677848029868 + 1e-8 + assert isinstance(Xt, make_df) + assert to_dict(Xt) == {"var": pytest.approx(expected)} + assert max(to_dict(Xt)["var"]) <= 0.10727677848029868 + 1e-8 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_both_ends_capping(make_df): - X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict={"var": 0.20857275540714884}, min_capping_dict={"var": -0.19661115230025186}, ) - Xt = transformer.fit_transform(X) + Xt = transformer.fit_transform(make_df(DATA)) # expected output expected = [ @@ -72,19 +70,18 @@ def test_both_ends_capping(make_df): -0.19661115230025186, 3 ) # test transform output - result = _to_dict(Xt) - assert result["var"] == pytest.approx(expected) + assert isinstance(Xt, make_df) + result = to_dict(Xt) + assert result == {"var": pytest.approx(expected)} assert max(result["var"]) <= 0.20857275540714884 + 1e-8 assert min(result["var"]) >= -0.19661115230025186 - 1e-8 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_left_tail_capping(make_df): - X = make_df(DATA) transformer = ArbitraryOutlierCapper( max_capping_dict=None, min_capping_dict={"var": -0.17486039103044} ) - Xt = transformer.fit_transform(X) + Xt = transformer.fit_transform(make_df(DATA)) # expected output expected = [max(v, -0.17486039103044) for v in DATA["var"]] @@ -100,32 +97,30 @@ def test_left_tail_capping(make_df): -0.17486039103044, 3 ) # test transform output - result = _to_dict(Xt) - assert result["var"] == pytest.approx(expected) + assert isinstance(Xt, make_df) + result = to_dict(Xt) + assert result == {"var": pytest.approx(expected)} assert min(result["var"]) >= -0.17486039103044 - 1e-8 -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_ignores_na_in_input_df(make_df): - X = make_df(DATA_NA) transformer = ArbitraryOutlierCapper( max_capping_dict=None, min_capping_dict={"Age": 20}, missing_values="ignore" ) - Xt = transformer.fit_transform(X) + Xt = transformer.fit_transform(make_df(DATA_NA)) # expected output - expected = [ - v if np.isnan(v) else max(v, 20) for v in DATA_NA["Age"] - ] + expected = [None if v is None else max(v, 20) for v in DATA_NA["Age"]] # test fit params assert transformer.max_capping_dict is None assert transformer.min_capping_dict == {"Age": 20} assert transformer.n_features_in_ == 5 # test transform output - result = _to_dict(Xt) - assert result["Age"] == pytest.approx(expected, nan_ok=True) - assert np.nanmin(result["Age"]) >= 20 + assert isinstance(Xt, make_df) + result = to_dict(Xt) + assert result["Age"] == expected + assert min(v for v in result["Age"] if v is not None) >= 20 def test_error_if_max_capping_dict_wrong_input(): @@ -152,29 +147,20 @@ def test_error_if_missing_values_not_bool(): ArbitraryOutlierCapper(missing_values="other") -@pytest.mark.parametrize("make_df", [pd.DataFrame, pl.DataFrame]) def test_fit_and_transform_raise_error_if_df_contains_na(make_df): - X = make_df(DATA) - data_na = dict(DATA) - var_na = list(DATA["var"]) - var_na[1] = np.nan - data_na["var"] = var_na - X_na = make_df(data_na) + data_na = {"var": list(DATA["var"])} + data_na["var"][1] = None # test case: when dataset contains na, fit method - with pytest.raises(ValueError): - transformer = ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044} - ) - transformer.fit(X_na) + transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.17486039103044}) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.fit(make_df(data_na)) # test case: when dataset contains na, transform method - with pytest.raises(ValueError): - transformer = ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044} - ) - transformer.fit(X) - transformer.transform(X_na) + transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.17486039103044}) + transformer.fit(make_df(DATA)) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.transform(make_df(data_na)) @pytest.mark.parametrize( @@ -183,7 +169,7 @@ def test_fit_and_transform_raise_error_if_df_contains_na(make_df): ) def test_error_if_missing_values_wrong_type(missing_values): msg = "missing_values takes only values 'raise' or 'ignore'" - with pytest.raises(ValueError, match=msg): + with pytest.raises(ValueError, match=re.escape(msg)): ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044}, missing_values="missing_values" + min_capping_dict={"var": -0.17486039103044}, missing_values=missing_values ) From 1c6fa6fd5921c7c0e65d04189c88c5de68725fee Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Tue, 15 Sep 2026 12:01:57 +0200 Subject: [PATCH 4/6] Use frame_to_dict after the shared helper rename in #1045 Co-Authored-By: Claude Opus 5 --- tests/test_outliers/test_arbitrary_capper.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index 9231c6856..ac6504f97 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -5,7 +5,7 @@ import pytest from feature_engine.outliers import ArbitraryOutlierCapper -from tests.backend_helpers import to_dict +from tests.backend_helpers import frame_to_dict DATA = {"var": np.random.RandomState(0).normal(0, 0.1, 20).tolist()} @@ -46,8 +46,8 @@ def test_right_end_capping(make_df): assert transformer.n_features_in_ == 1 # test transform output assert isinstance(Xt, make_df) - assert to_dict(Xt) == {"var": pytest.approx(expected)} - assert max(to_dict(Xt)["var"]) <= 0.10727677848029868 + 1e-8 + assert frame_to_dict(Xt) == {"var": pytest.approx(expected)} + assert max(frame_to_dict(Xt)["var"]) <= 0.10727677848029868 + 1e-8 def test_both_ends_capping(make_df): @@ -71,7 +71,7 @@ def test_both_ends_capping(make_df): ) # test transform output assert isinstance(Xt, make_df) - result = to_dict(Xt) + result = frame_to_dict(Xt) assert result == {"var": pytest.approx(expected)} assert max(result["var"]) <= 0.20857275540714884 + 1e-8 assert min(result["var"]) >= -0.19661115230025186 - 1e-8 @@ -98,7 +98,7 @@ def test_left_tail_capping(make_df): ) # test transform output assert isinstance(Xt, make_df) - result = to_dict(Xt) + result = frame_to_dict(Xt) assert result == {"var": pytest.approx(expected)} assert min(result["var"]) >= -0.17486039103044 - 1e-8 @@ -118,7 +118,7 @@ def test_ignores_na_in_input_df(make_df): assert transformer.n_features_in_ == 5 # test transform output assert isinstance(Xt, make_df) - result = to_dict(Xt) + result = frame_to_dict(Xt) assert result["Age"] == expected assert min(v for v in result["Age"] if v is not None) >= 20 From 9c6f168c0ea5ea9a1fe31239add9457a1e8b9f61 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 19 Sep 2026 09:00:23 +0200 Subject: [PATCH 5/6] Align ArbitraryOutlierCapper and its tests with the repo conventions Co-Authored-By: Claude Opus 5 --- feature_engine/outliers/artbitrary.py | 60 +++-- tests/test_outliers/test_arbitrary_capper.py | 228 +++++++++---------- 2 files changed, 131 insertions(+), 157 deletions(-) diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index ba8febf99..98e983c8a 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -119,16 +119,24 @@ def __init__( missing_values: str = "raise", ) -> None: - if not max_capping_dict and not min_capping_dict: + _check_numerical_dict(max_capping_dict) + _check_numerical_dict(min_capping_dict) + + if (max_capping_dict is None or len(max_capping_dict) == 0) and ( + min_capping_dict is None or len(min_capping_dict) == 0 + ): raise ValueError( "Please provide at least 1 dictionary with the capping values." ) - if missing_values not in ["raise", "ignore"]: - raise ValueError("missing_values takes only values 'raise' or 'ignore'") - - _check_numerical_dict(max_capping_dict) - _check_numerical_dict(min_capping_dict) + if not isinstance(missing_values, str) or missing_values not in [ + "raise", + "ignore", + ]: + raise ValueError( + "missing_values must be 'raise' or 'ignore'. " + f"Got {missing_values} instead." + ) self.max_capping_dict = max_capping_dict self.min_capping_dict = min_capping_dict @@ -148,37 +156,25 @@ def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ nw_X = check_X(X) - # find variables to be capped - if self.min_capping_dict is None and self.max_capping_dict: - self.variables_ = [x for x in self.max_capping_dict.keys()] - elif self.max_capping_dict is None and self.min_capping_dict: - self.variables_ = [x for x in self.min_capping_dict.keys()] - elif self.min_capping_dict and self.max_capping_dict: - tmp = self.min_capping_dict.copy() - tmp.update(self.max_capping_dict) - self.variables_ = [x for x in tmp.keys()] + if self.max_capping_dict is None: + self.right_tail_caps_ = {} + else: + self.right_tail_caps_ = self.max_capping_dict - if self.missing_values == "raise": - # check if dataset contains na - _check_contains_na(X, self.variables_) - _check_contains_inf(X, self.variables_) + if self.min_capping_dict is None: + self.left_tail_caps_ = {} + else: + self.left_tail_caps_ = self.min_capping_dict - # find or check for numerical variables - self.variables_ = check_numerical_variables(X, self.variables_) + variables = list({**self.left_tail_caps_, **self.right_tail_caps_}) - if self.max_capping_dict is not None: - self.right_tail_caps_ = self.max_capping_dict - else: - self.right_tail_caps_ = {} + if self.missing_values == "raise": + _check_contains_na(X, variables) + _check_contains_inf(X, variables) - if self.min_capping_dict is not None: - self.left_tail_caps_ = self.min_capping_dict - else: - self.left_tail_caps_ = {} + self.variables_ = check_numerical_variables(X, variables) - # list() normalises both a narwhals `.columns` (already a list) and a - # pandas Index to a plain list. - self.feature_names_in_ = list(nw_X.columns) + self.feature_names_in_ = nw_X.columns self.n_features_in_ = nw_X.shape[1] return self diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index ac6504f97..62ced5701 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -1,4 +1,3 @@ -import datetime import re import numpy as np @@ -7,169 +6,148 @@ from feature_engine.outliers import ArbitraryOutlierCapper from tests.backend_helpers import frame_to_dict -DATA = {"var": np.random.RandomState(0).normal(0, 0.1, 20).tolist()} - -DATA_NA = { - "Name": ["tom", "nick", "krish", "jack", "tom", "eric"], - "City": ["London", "Manchester", "Liverpool", "Bristol", "Manchester", "Liverpool"], - "Age": [20.0, 21.0, 19.0, 18.0, None, 41.0], - "Marks": [0.9, 0.8, 0.7, 0.6, 0.5, 0.6], - "dob": [datetime.datetime(2020, 2, 24, 0, i) for i in range(6)], -} - MSG_NA = ( "Some of the variables in the dataset contain NaN. Check and " "remove those before using this transformer." ) -def test_right_end_capping(make_df): - transformer = ArbitraryOutlierCapper( - max_capping_dict={"var": 0.10727677848029868}, min_capping_dict=None - ) - Xt = transformer.fit_transform(make_df(DATA)) +# init parameters +@pytest.mark.parametrize("param", ["max_capping_dict", "min_capping_dict"]) +@pytest.mark.parametrize("value", ["other", 1, ["var"], ("var", 1)]) +def test_error_if_capping_dict_not_dict(param, value): + msg = f"The parameter can only take a dictionary or None. Got {value} instead." + with pytest.raises(TypeError, match=re.escape(msg)): + ArbitraryOutlierCapper(**{param: value}) - # expected output - expected = [min(v, 0.10727677848029868) for v in DATA["var"]] - # test init params - assert np.round(transformer.max_capping_dict["var"], 3) == np.round( - 0.10727677848029868, 3 +@pytest.mark.parametrize("param", ["max_capping_dict", "min_capping_dict"]) +@pytest.mark.parametrize("value", [{"var": "a"}, {"var": None}, {"a": 1, "b": [2]}]) +def test_error_if_capping_dict_values_not_numerical(param, value): + msg = ( + "All values in the dictionary must be integer or float. " + f"Got {value} instead." ) - assert transformer.min_capping_dict is None - assert transformer.variables_ == ["var"] - # test fit attrs - assert np.round(transformer.right_tail_caps_["var"], 3) == np.round( - 0.10727677848029868, 3 - ) - assert transformer.left_tail_caps_ == {} - assert transformer.n_features_in_ == 1 - # test transform output - assert isinstance(Xt, make_df) - assert frame_to_dict(Xt) == {"var": pytest.approx(expected)} - assert max(frame_to_dict(Xt)["var"]) <= 0.10727677848029868 + 1e-8 + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper(**{param: value}) -def test_both_ends_capping(make_df): - transformer = ArbitraryOutlierCapper( - max_capping_dict={"var": 0.20857275540714884}, - min_capping_dict={"var": -0.19661115230025186}, - ) - Xt = transformer.fit_transform(make_df(DATA)) +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict", + [(None, None), ({}, None), (None, {}), ({}, {})], +) +def test_error_if_no_capping_values(max_capping_dict, min_capping_dict): + msg = "Please provide at least 1 dictionary with the capping values." + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper( + max_capping_dict=max_capping_dict, min_capping_dict=min_capping_dict + ) - # expected output - expected = [ - min(max(v, -0.19661115230025186), 0.20857275540714884) for v in DATA["var"] - ] - # test fit params - assert np.round(transformer.right_tail_caps_["var"], 3) == np.round( - 0.20857275540714884, 3 - ) - assert np.round(transformer.left_tail_caps_["var"], 3) == np.round( - -0.19661115230025186, 3 +@pytest.mark.parametrize( + "missing_values", ["HOLA", "Raise", 1, True, None, ["raise"], {"key": "raise"}] +) +def test_error_if_missing_values_not_permitted(missing_values): + msg = ( + "missing_values must be 'raise' or 'ignore'. " + f"Got {missing_values} instead." ) - # test transform output - assert isinstance(Xt, make_df) - result = frame_to_dict(Xt) - assert result == {"var": pytest.approx(expected)} - assert max(result["var"]) <= 0.20857275540714884 + 1e-8 - assert min(result["var"]) >= -0.19661115230025186 - 1e-8 + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper( + min_capping_dict={"var": -0.15}, missing_values=missing_values + ) -def test_left_tail_capping(make_df): +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict, missing_values", + [ + ({"var": 0.1}, None, "raise"), + (None, {"var": -0.15}, "ignore"), + ({"var": 0.1}, {"var": -0.15, "other": 2}, "raise"), + ({"var": 1}, {}, "ignore"), + ], +) +def test_init_param_assignment(max_capping_dict, min_capping_dict, missing_values): transformer = ArbitraryOutlierCapper( - max_capping_dict=None, min_capping_dict={"var": -0.17486039103044} + max_capping_dict=max_capping_dict, + min_capping_dict=min_capping_dict, + missing_values=missing_values, ) - Xt = transformer.fit_transform(make_df(DATA)) + assert transformer.max_capping_dict == max_capping_dict + assert transformer.min_capping_dict == min_capping_dict + assert transformer.missing_values == missing_values - # expected output - expected = [max(v, -0.17486039103044) for v in DATA["var"]] - # test init param - assert transformer.max_capping_dict is None - assert np.round(transformer.min_capping_dict["var"], 3) == np.round( - -0.17486039103044, 3 - ) - # test fit attr - assert transformer.right_tail_caps_ == {} - assert np.round(transformer.left_tail_caps_["var"], 3) == np.round( - -0.17486039103044, 3 +# fit and transform +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict", + [({"var": 0.1}, None), (None, {"var": -0.15}), ({"var": 0.1}, {"var": -0.15})], +) +def test_capping(make_df, data_normal_dist, max_capping_dict, min_capping_dict): + transformer = ArbitraryOutlierCapper( + max_capping_dict=max_capping_dict, min_capping_dict=min_capping_dict ) - # test transform output + Xt = transformer.fit_transform(make_df(data_normal_dist)) + + upper = np.inf if max_capping_dict is None else max_capping_dict["var"] + lower = -np.inf if min_capping_dict is None else min_capping_dict["var"] + expected = [min(max(v, lower), upper) for v in data_normal_dist["var"]] + + assert transformer.right_tail_caps_ == (max_capping_dict or {}) + assert transformer.left_tail_caps_ == (min_capping_dict or {}) + assert transformer.variables_ == ["var"] + assert transformer.feature_names_in_ == ["var"] + assert transformer.n_features_in_ == 1 assert isinstance(Xt, make_df) - result = frame_to_dict(Xt) - assert result == {"var": pytest.approx(expected)} - assert min(result["var"]) >= -0.17486039103044 - 1e-8 + assert frame_to_dict(Xt) == {"var": pytest.approx(expected)} -def test_ignores_na_in_input_df(make_df): +def test_variables_are_taken_from_both_dicts(make_df): + X = make_df({"a": [0, 5, 10], "b": [0, 5, 10], "c": [0, 5, 10]}) transformer = ArbitraryOutlierCapper( - max_capping_dict=None, min_capping_dict={"Age": 20}, missing_values="ignore" + max_capping_dict={"a": 8}, min_capping_dict={"b": 2, "a": 1} ) - Xt = transformer.fit_transform(make_df(DATA_NA)) - - # expected output - expected = [None if v is None else max(v, 20) for v in DATA_NA["Age"]] + Xt = transformer.fit_transform(X) - # test fit params - assert transformer.max_capping_dict is None - assert transformer.min_capping_dict == {"Age": 20} - assert transformer.n_features_in_ == 5 - # test transform output + assert transformer.variables_ == ["b", "a"] assert isinstance(Xt, make_df) - result = frame_to_dict(Xt) - assert result["Age"] == expected - assert min(v for v in result["Age"] if v is not None) >= 20 - + assert frame_to_dict(Xt) == {"a": [1, 5, 8], "b": [2, 5, 10], "c": [0, 5, 10]} -def test_error_if_max_capping_dict_wrong_input(): - with pytest.raises(TypeError): - ArbitraryOutlierCapper(max_capping_dict="other") - with pytest.raises(ValueError): - ArbitraryOutlierCapper(max_capping_dict={"a": "a"}) +def test_empty_dict_is_ignored(make_df): + X = make_df({"a": [0, 5, 10], "b": [0, 5, 10]}) + transformer = ArbitraryOutlierCapper(max_capping_dict={"a": 8}, min_capping_dict={}) + Xt = transformer.fit_transform(X) -def test_error_if_min_capping_dict_wrong_input(): - with pytest.raises(TypeError): - ArbitraryOutlierCapper(min_capping_dict="other") - with pytest.raises(ValueError): - ArbitraryOutlierCapper(min_capping_dict={"a": "a"}) - + assert transformer.variables_ == ["a"] + assert transformer.left_tail_caps_ == {} + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt) == {"a": [0, 5, 8], "b": [0, 5, 10]} -def test_error_if_both_capping_dicts_are_none(): - with pytest.raises(ValueError): - ArbitraryOutlierCapper(min_capping_dict=None, max_capping_dict=None) +def test_ignores_na_in_input_df(make_df, data_na): + transformer = ArbitraryOutlierCapper( + min_capping_dict={"Age": 21}, missing_values="ignore" + ) + Xt = transformer.fit_transform(make_df(data_na)) -def test_error_if_missing_values_not_bool(): - with pytest.raises(ValueError): - ArbitraryOutlierCapper(missing_values="other") + expected = [None if v is None else max(v, 21) for v in data_na["Age"]] + assert transformer.n_features_in_ == 5 + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt)["Age"] == expected -def test_fit_and_transform_raise_error_if_df_contains_na(make_df): - data_na = {"var": list(DATA["var"])} - data_na["var"][1] = None - # test case: when dataset contains na, fit method - transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.17486039103044}) +def test_fit_raises_error_if_df_contains_na(make_df, data_na): + transformer = ArbitraryOutlierCapper(min_capping_dict={"Age": 21}) with pytest.raises(ValueError, match=re.escape(MSG_NA)): transformer.fit(make_df(data_na)) - # test case: when dataset contains na, transform method - transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.17486039103044}) - transformer.fit(make_df(DATA)) + +def test_transform_raises_error_if_df_contains_na(make_df, data_normal_dist): + data_na = {"var": list(data_normal_dist["var"])} + data_na["var"][1] = None + transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.15}) + transformer.fit(make_df(data_normal_dist)) with pytest.raises(ValueError, match=re.escape(MSG_NA)): transformer.transform(make_df(data_na)) - - -@pytest.mark.parametrize( - "missing_values", - ["HOLA", 1, True, {"key1": "value1", "key2": "value2", "key3": "value3"}], -) -def test_error_if_missing_values_wrong_type(missing_values): - msg = "missing_values takes only values 'raise' or 'ignore'" - with pytest.raises(ValueError, match=re.escape(msg)): - ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044}, missing_values=missing_values - ) From 4c32ed15a221b1df9168b46b86dc9ec1c5fbf386 Mon Sep 17 00:00:00 2001 From: Soledad Galli Date: Sat, 19 Sep 2026 09:43:15 +0200 Subject: [PATCH 6/6] Give the expected caps explicitly in the ArbitraryOutlierCapper capping test Co-Authored-By: Claude Opus 5 --- tests/test_outliers/test_arbitrary_capper.py | 28 ++++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index 62ced5701..641daa5ce 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -80,21 +80,33 @@ def test_init_param_assignment(max_capping_dict, min_capping_dict, missing_value # fit and transform @pytest.mark.parametrize( - "max_capping_dict, min_capping_dict", - [({"var": 0.1}, None), (None, {"var": -0.15}), ({"var": 0.1}, {"var": -0.15})], + "max_capping_dict, min_capping_dict, right_tail_caps, left_tail_caps", + [ + ({"var": 0.1}, None, {"var": 0.1}, {}), + (None, {"var": -0.15}, {}, {"var": -0.15}), + ({"var": 0.1}, {"var": -0.15}, {"var": 0.1}, {"var": -0.15}), + ], ) -def test_capping(make_df, data_normal_dist, max_capping_dict, min_capping_dict): +def test_capping( + make_df, + data_normal_dist, + max_capping_dict, + min_capping_dict, + right_tail_caps, + left_tail_caps, +): transformer = ArbitraryOutlierCapper( max_capping_dict=max_capping_dict, min_capping_dict=min_capping_dict ) Xt = transformer.fit_transform(make_df(data_normal_dist)) - upper = np.inf if max_capping_dict is None else max_capping_dict["var"] - lower = -np.inf if min_capping_dict is None else min_capping_dict["var"] - expected = [min(max(v, lower), upper) for v in data_normal_dist["var"]] + # a tail without a limit is not capped + upper = right_tail_caps.get("var", np.inf) + lower = left_tail_caps.get("var", -np.inf) + expected = np.clip(data_normal_dist["var"], lower, upper).tolist() - assert transformer.right_tail_caps_ == (max_capping_dict or {}) - assert transformer.left_tail_caps_ == (min_capping_dict or {}) + assert transformer.right_tail_caps_ == right_tail_caps + assert transformer.left_tail_caps_ == left_tail_caps assert transformer.variables_ == ["var"] assert transformer.feature_names_in_ == ["var"] assert transformer.n_features_in_ == 1