diff --git a/docs/user_guide/outliers/ArbitraryOutlierCapper.rst b/docs/user_guide/outliers/ArbitraryOutlierCapper.rst index 70153b25b..ce916e9c4 100644 --- a/docs/user_guide/outliers/ArbitraryOutlierCapper.rst +++ b/docs/user_guide/outliers/ArbitraryOutlierCapper.rst @@ -96,6 +96,44 @@ values: dtype: float64 +With polars +----------- + +:class:`ArbitraryOutlierCapper()` works in the same way with a polars dataframe: + +.. code:: python + + import polars as pl + from feature_engine.outliers import ArbitraryOutlierCapper + + df = pl.DataFrame({ + "age": [20.0, 21.0, 19.0, 45.0, 67.0, 18.0, 90.0, 34.0, 55.0, 23.0], + "fare": [7.5, 8.0, 71.3, 13.0, 30.5, 7.9, 512.3, 26.0, 15.5, 8.6], + }) + + capper = ArbitraryOutlierCapper( + max_capping_dict={"age": 50, "fare": 200}, + min_capping_dict=None, + ) + + capper.fit(df) + Xt = capper.transform(df) + + print(Xt.select(["age", "fare"]).max()) + +The resulting maximum values, capped at the values we entered in the dictionary: + +.. code:: text + + shape: (1, 2) + ┌──────┬───────┐ + │ age ┆ fare │ + │ --- ┆ --- │ + │ f64 ┆ f64 │ + ╞══════╪═══════╡ + │ 50.0 ┆ 200.0 │ + └──────┴───────┘ + Additional resources -------------------- diff --git a/feature_engine/outliers/artbitrary.py b/feature_engine/outliers/artbitrary.py index 6088520da..98e983c8a 100644 --- a/feature_engine/outliers/artbitrary.py +++ b/feature_engine/outliers/artbitrary.py @@ -4,7 +4,7 @@ from typing import Optional -import pandas as pd +from narwhals.typing import IntoDataFrame, IntoSeries from feature_engine._check_init_parameters.check_input_dictionary import ( _check_numerical_dict, @@ -119,80 +119,78 @@ def __init__( missing_values: str = "raise", ) -> None: - if not max_capping_dict and not min_capping_dict: + _check_numerical_dict(max_capping_dict) + _check_numerical_dict(min_capping_dict) + + if (max_capping_dict is None or len(max_capping_dict) == 0) and ( + min_capping_dict is None or len(min_capping_dict) == 0 + ): raise ValueError( "Please provide at least 1 dictionary with the capping values." ) - if missing_values not in ["raise", "ignore"]: - raise ValueError("missing_values takes only values 'raise' or 'ignore'") - - _check_numerical_dict(max_capping_dict) - _check_numerical_dict(min_capping_dict) + if not isinstance(missing_values, str) or missing_values not in [ + "raise", + "ignore", + ]: + raise ValueError( + "missing_values must be 'raise' or 'ignore'. " + f"Got {missing_values} instead." + ) self.max_capping_dict = max_capping_dict self.min_capping_dict = min_capping_dict self.missing_values = missing_values - def fit(self, X: pd.DataFrame, y: Optional[pd.Series] = None): + def fit(self, X: IntoDataFrame, y: Optional[IntoSeries] = None): """ This transformer does not learn any parameter. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The training input samples. - y: pandas Series, default=None + y: Series, default=None y is not needed in this transformer. You can pass y or None. """ - X = check_X(X) - - # find variables to be capped - if self.min_capping_dict is None and self.max_capping_dict: - self.variables_ = [x for x in self.max_capping_dict.keys()] - elif self.max_capping_dict is None and self.min_capping_dict: - self.variables_ = [x for x in self.min_capping_dict.keys()] - elif self.min_capping_dict and self.max_capping_dict: - tmp = self.min_capping_dict.copy() - tmp.update(self.max_capping_dict) - self.variables_ = [x for x in tmp.keys()] - - if self.missing_values == "raise": - # check if dataset contains na - _check_contains_na(X, self.variables_) - _check_contains_inf(X, self.variables_) - - # find or check for numerical variables - self.variables_ = check_numerical_variables(X, self.variables_) + nw_X = check_X(X) - if self.max_capping_dict is not None: - self.right_tail_caps_ = self.max_capping_dict - else: + if self.max_capping_dict is None: self.right_tail_caps_ = {} - - if self.min_capping_dict is not None: - self.left_tail_caps_ = self.min_capping_dict else: + self.right_tail_caps_ = self.max_capping_dict + + if self.min_capping_dict is None: self.left_tail_caps_ = {} + else: + self.left_tail_caps_ = self.min_capping_dict + + variables = list({**self.left_tail_caps_, **self.right_tail_caps_}) + + if self.missing_values == "raise": + _check_contains_na(X, variables) + _check_contains_inf(X, variables) + + self.variables_ = check_numerical_variables(X, variables) - self.feature_names_in_ = X.columns.to_list() - self.n_features_in_ = X.shape[1] + self.feature_names_in_ = nw_X.columns + self.n_features_in_ = nw_X.shape[1] return self - def transform(self, X: pd.DataFrame) -> pd.DataFrame: + def transform(self, X: IntoDataFrame) -> IntoDataFrame: """ Cap the variable values. Parameters ---------- - X: pandas dataframe of shape = [n_samples, n_features] + X: dataframe of shape = [n_samples, n_features] The data to be transformed. Returns ------- - X_new: pandas dataframe of shape = [n_samples, n_features] + X_new: dataframe of shape = [n_samples, n_features] The dataframe with the capped variables. """ return super()._transform(X) diff --git a/tests/test_outliers/test_arbitrary_capper.py b/tests/test_outliers/test_arbitrary_capper.py index 5cba357b8..641daa5ce 100644 --- a/tests/test_outliers/test_arbitrary_capper.py +++ b/tests/test_outliers/test_arbitrary_capper.py @@ -1,176 +1,165 @@ +import re + import numpy as np -import pandas as pd import pytest from feature_engine.outliers import ArbitraryOutlierCapper +from tests.backend_helpers import frame_to_dict +MSG_NA = ( + "Some of the variables in the dataset contain NaN. Check and " + "remove those before using this transformer." +) -def test_right_end_capping(df_normal_dist): - # test case 1: right end capping - transformer = ArbitraryOutlierCapper( - max_capping_dict={"var": 0.10727677848029868}, min_capping_dict=None - ) - X = transformer.fit_transform(df_normal_dist) - - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] > 0.10727677848029868, 0.10727677848029868, df_transf["var"] - ) - # test init params - assert np.round(transformer.max_capping_dict["var"], 3) == np.round( - 0.10727677848029868, 3 - ) - assert transformer.min_capping_dict is None - assert transformer.variables_ == ["var"] - # test fit attrs - assert np.round(transformer.right_tail_caps_["var"], 3) == np.round( - 0.10727677848029868, 3 - ) - assert transformer.left_tail_caps_ == {} - assert transformer.n_features_in_ == 1 - # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].max(), 3) <= np.round(0.10727677848029868, 3) - assert np.round(df_normal_dist["var"].max(), 3) > np.round(0.10727677848029868, 3) +# init parameters +@pytest.mark.parametrize("param", ["max_capping_dict", "min_capping_dict"]) +@pytest.mark.parametrize("value", ["other", 1, ["var"], ("var", 1)]) +def test_error_if_capping_dict_not_dict(param, value): + msg = f"The parameter can only take a dictionary or None. Got {value} instead." + with pytest.raises(TypeError, match=re.escape(msg)): + ArbitraryOutlierCapper(**{param: value}) -def test_both_ends_capping(df_normal_dist): - # test case 2: both tails - transformer = ArbitraryOutlierCapper( - max_capping_dict={"var": 0.20857275540714884}, - min_capping_dict={"var": -0.19661115230025186}, +@pytest.mark.parametrize("param", ["max_capping_dict", "min_capping_dict"]) +@pytest.mark.parametrize("value", [{"var": "a"}, {"var": None}, {"a": 1, "b": [2]}]) +def test_error_if_capping_dict_values_not_numerical(param, value): + msg = ( + "All values in the dictionary must be integer or float. " + f"Got {value} instead." ) - X = transformer.fit_transform(df_normal_dist) + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper(**{param: value}) - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] > 0.20857275540714884, 0.20857275540714884, df_transf["var"] - ) - df_transf["var"] = np.where( - df_transf["var"] < -0.19661115230025186, -0.19661115230025186, df_transf["var"] - ) - # test fit params - assert np.round(transformer.right_tail_caps_["var"], 3) == np.round( - 0.20857275540714884, 3 - ) - assert np.round(transformer.left_tail_caps_["var"], 3) == np.round( - -0.19661115230025186, 3 - ) - # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].max(), 3) <= np.round(0.20857275540714884, 3) - assert np.round(X["var"].min(), 3) >= np.round(-0.19661115230025186, 3) - assert np.round(df_normal_dist["var"].max(), 3) > np.round(0.20857275540714884, 3) - assert np.round(df_normal_dist["var"].min(), 3) < np.round(-0.19661115230025186, 3) +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict", + [(None, None), ({}, None), (None, {}), ({}, {})], +) +def test_error_if_no_capping_values(max_capping_dict, min_capping_dict): + msg = "Please provide at least 1 dictionary with the capping values." + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper( + max_capping_dict=max_capping_dict, min_capping_dict=min_capping_dict + ) -def test_left_tail_capping(df_normal_dist): - # test case 3: left tail - transformer = ArbitraryOutlierCapper( - max_capping_dict=None, min_capping_dict={"var": -0.17486039103044} +@pytest.mark.parametrize( + "missing_values", ["HOLA", "Raise", 1, True, None, ["raise"], {"key": "raise"}] +) +def test_error_if_missing_values_not_permitted(missing_values): + msg = ( + "missing_values must be 'raise' or 'ignore'. " + f"Got {missing_values} instead." ) - X = transformer.fit_transform(df_normal_dist) + with pytest.raises(ValueError, match=re.escape(msg)): + ArbitraryOutlierCapper( + min_capping_dict={"var": -0.15}, missing_values=missing_values + ) - # expected output - df_transf = df_normal_dist.copy() - df_transf["var"] = np.where( - df_transf["var"] < -0.17486039103044, -0.17486039103044, df_transf["var"] - ) - # test init param - assert transformer.max_capping_dict is None - assert np.round(transformer.min_capping_dict["var"], 3) == np.round( - -0.17486039103044, 3 - ) - # test fit attr - assert transformer.right_tail_caps_ == {} - assert np.round(transformer.left_tail_caps_["var"], 3) == np.round( - -0.17486039103044, 3 +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict, missing_values", + [ + ({"var": 0.1}, None, "raise"), + (None, {"var": -0.15}, "ignore"), + ({"var": 0.1}, {"var": -0.15, "other": 2}, "raise"), + ({"var": 1}, {}, "ignore"), + ], +) +def test_init_param_assignment(max_capping_dict, min_capping_dict, missing_values): + transformer = ArbitraryOutlierCapper( + max_capping_dict=max_capping_dict, + min_capping_dict=min_capping_dict, + missing_values=missing_values, ) - # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert np.round(X["var"].min(), 3) >= np.round(-0.17486039103044, 3) - assert np.round(df_normal_dist["var"].min(), 3) < np.round(-0.17486039103044, 3) + assert transformer.max_capping_dict == max_capping_dict + assert transformer.min_capping_dict == min_capping_dict + assert transformer.missing_values == missing_values -def test_ignores_na_in_input_df(df_na): - # test case 4: dataset contains na and transformer is asked to ignore them +# fit and transform +@pytest.mark.parametrize( + "max_capping_dict, min_capping_dict, right_tail_caps, left_tail_caps", + [ + ({"var": 0.1}, None, {"var": 0.1}, {}), + (None, {"var": -0.15}, {}, {"var": -0.15}), + ({"var": 0.1}, {"var": -0.15}, {"var": 0.1}, {"var": -0.15}), + ], +) +def test_capping( + make_df, + data_normal_dist, + max_capping_dict, + min_capping_dict, + right_tail_caps, + left_tail_caps, +): transformer = ArbitraryOutlierCapper( - max_capping_dict=None, min_capping_dict={"Age": 20}, missing_values="ignore" + max_capping_dict=max_capping_dict, min_capping_dict=min_capping_dict ) - X = transformer.fit_transform(df_na) + Xt = transformer.fit_transform(make_df(data_normal_dist)) - # expected output - df_transf = df_na.copy() - df_transf["Age"] = np.where(df_transf["Age"] < 20, 20, df_transf["Age"]) + # a tail without a limit is not capped + upper = right_tail_caps.get("var", np.inf) + lower = left_tail_caps.get("var", -np.inf) + expected = np.clip(data_normal_dist["var"], lower, upper).tolist() - # test fit params - assert transformer.max_capping_dict is None - assert transformer.min_capping_dict == {"Age": 20} - assert transformer.n_features_in_ == 6 - # test transform output - pd.testing.assert_frame_equal(X, df_transf) - assert X["Age"].min() >= 20 - assert df_na["Age"].min() < 20 + assert transformer.right_tail_caps_ == right_tail_caps + assert transformer.left_tail_caps_ == left_tail_caps + assert transformer.variables_ == ["var"] + assert transformer.feature_names_in_ == ["var"] + assert transformer.n_features_in_ == 1 + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt) == {"var": pytest.approx(expected)} -def test_error_if_max_capping_dict_wrong_input(): - with pytest.raises(TypeError): - ArbitraryOutlierCapper(max_capping_dict="other") - with pytest.raises(ValueError): - ArbitraryOutlierCapper(max_capping_dict={"a": "a"}) +def test_variables_are_taken_from_both_dicts(make_df): + X = make_df({"a": [0, 5, 10], "b": [0, 5, 10], "c": [0, 5, 10]}) + transformer = ArbitraryOutlierCapper( + max_capping_dict={"a": 8}, min_capping_dict={"b": 2, "a": 1} + ) + Xt = transformer.fit_transform(X) + assert transformer.variables_ == ["b", "a"] + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt) == {"a": [1, 5, 8], "b": [2, 5, 10], "c": [0, 5, 10]} -def test_error_if_min_capping_dict_wrong_input(): - with pytest.raises(TypeError): - ArbitraryOutlierCapper(min_capping_dict="other") - with pytest.raises(ValueError): - ArbitraryOutlierCapper(min_capping_dict={"a": "a"}) +def test_empty_dict_is_ignored(make_df): + X = make_df({"a": [0, 5, 10], "b": [0, 5, 10]}) + transformer = ArbitraryOutlierCapper(max_capping_dict={"a": 8}, min_capping_dict={}) + Xt = transformer.fit_transform(X) -def test_error_if_both_capping_dicts_are_none(): - with pytest.raises(ValueError): - ArbitraryOutlierCapper(min_capping_dict=None, max_capping_dict=None) + assert transformer.variables_ == ["a"] + assert transformer.left_tail_caps_ == {} + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt) == {"a": [0, 5, 8], "b": [0, 5, 10]} -def test_error_if_missing_values_not_bool(): - with pytest.raises(ValueError): - ArbitraryOutlierCapper(missing_values="other") +def test_ignores_na_in_input_df(make_df, data_na): + transformer = ArbitraryOutlierCapper( + min_capping_dict={"Age": 21}, missing_values="ignore" + ) + Xt = transformer.fit_transform(make_df(data_na)) + expected = [None if v is None else max(v, 21) for v in data_na["Age"]] -def test_fit_and_transform_raise_error_if_df_contains_na(df_normal_dist): - df_na = df_normal_dist.copy() - df_na.loc[1, "var"] = np.nan + assert transformer.n_features_in_ == 5 + assert isinstance(Xt, make_df) + assert frame_to_dict(Xt)["Age"] == expected - # test case 5: when dataset contains na, fit method - with pytest.raises(ValueError): - transformer = ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044} - ) - transformer.fit(df_na) - # test case 6: when dataset contains na, transform method - with pytest.raises(ValueError): - transformer = ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044} - ) - transformer.fit(df_normal_dist) - transformer.transform(df_na) +def test_fit_raises_error_if_df_contains_na(make_df, data_na): + transformer = ArbitraryOutlierCapper(min_capping_dict={"Age": 21}) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.fit(make_df(data_na)) -@pytest.mark.parametrize( - "missing_values", - ["HOLA", 1, True, {"key1": "value1", "key2": "value2", "key3": "value3"}], -) -def test_error_if_missing_values_wrong_type(missing_values): - msg = "missing_values takes only values 'raise' or 'ignore'" - with pytest.raises(ValueError) as record: - ArbitraryOutlierCapper( - min_capping_dict={"var": -0.17486039103044}, missing_values="missing_values" - ) - # check that error message matches - assert str(record.value) == msg +def test_transform_raises_error_if_df_contains_na(make_df, data_normal_dist): + data_na = {"var": list(data_normal_dist["var"])} + data_na["var"][1] = None + transformer = ArbitraryOutlierCapper(min_capping_dict={"var": -0.15}) + transformer.fit(make_df(data_normal_dist)) + with pytest.raises(ValueError, match=re.escape(MSG_NA)): + transformer.transform(make_df(data_na))