import pandas as pd
from woodwork.logical_types import Boolean, BooleanNullable
def remove_highly_null_features(feature_matrix, features=None, pct_null_threshold=0.95):
"""移除特征矩阵中空值比例高于设定阈值的列.
Args:
feature_matrix (:class:`pd.DataFrame`): 列名为特征名、行名为实例的DataFrame.
features (list[:class:`featuretools.FeatureBase`] 或 list[str], 可选): 要选择的特征列表.
pct_null_threshold (float): 如果输入特征中NaN值的百分比超过此数值,该特征将被视为高比例空值.默认为0.95.
Returns:
pd.DataFrame, list[:class:`.FeatureBase`]:
特征矩阵和生成的特征定义列表.与dfs输出格式一致.
如果未提供特征列表作为输入,则不会返回特征列表.
"""
if pct_null_threshold < 0 or pct_null_threshold > 1:
raise ValueError(
"pct_null_threshold must be a float between 0 and 1, inclusive.",
)
percent_null_by_col = (feature_matrix.isnull().mean()).to_dict()
if pct_null_threshold == 0.0:
keep = [
f_name
for f_name, pct_null in percent_null_by_col.items()
if pct_null <= pct_null_threshold
]
else:
keep = [
f_name
for f_name, pct_null in percent_null_by_col.items()
if pct_null < pct_null_threshold
]
return _apply_feature_selection(keep, feature_matrix, features)
def remove_single_value_features(
feature_matrix,
features=None,
count_nan_as_value=False,
):
""" 移除特征矩阵中所有值相同的列.
Args:
feature_matrix (:class:`pd.DataFrame`): 列名为特征名称、行名为实例的DataFrame.
features (list[:class:`featuretools.FeatureBase`] 或 list[str], 可选): 要选择的特征列表.
count_nan_as_value (bool): 如果为True,缺失值将被视为其自身的唯一值.
如果设置为False,具有一个唯一值且其他数据全部缺失的特征将从特征矩阵中移除.默认为False.
Returns:
pd.DataFrame, list[:class:`.FeatureBase`]:
特征矩阵和生成的特征定义列表.
与dfs输出匹配.
如果未提供特征列表作为输入,则不会返回特征列表.
"""
unique_counts_by_col = feature_matrix.nunique(
dropna=not count_nan_as_value,
).to_dict()
keep = [
f_name
for f_name, unique_count in unique_counts_by_col.items()
if unique_count > 1
]
return _apply_feature_selection(keep, feature_matrix, features)
def remove_highly_correlated_features(
feature_matrix,
features=None,
pct_corr_threshold=0.95,
features_to_check=None,
features_to_keep=None,
):
""" 移除特征矩阵中与其他列高度相关的列.
注意:
我们假设,对于一对特征,在``dfs``生成的特征矩阵中,位置更靠右的特征是更复杂的特征.
如果特征矩阵的列顺序与``dfs``生成的顺序不同,则此假设不成立.
Args:
feature_matrix (:class:`pd.DataFrame`):DataFrame,其列名为特征名称,行名为实例.
如果未初始化Woodwork,将执行Woodwork初始化,这可能导致与Featuretools创建的原始特征矩阵中的类型略有不同.
features (list[:class:`featuretools.FeatureBase`] 或 list[str], 可选):
要选择的特征列表.
pct_corr_threshold (float):相关性阈值,高于此值被视为高度相关.默认为0.95.
features_to_check (list[str], 可选):要检查是否存在任何高度相关对的列名列表.
不会检查任何其他列,这意味着唯一可能被移除的列在此列表中.如果为空,默认检查所有列.
features_to_keep (list[str], 可选):即使与其他列相关也要保留的列名列表.
如果为空,所有列都将作为移除的候选.
Returns:
pd.DataFrame, list[:class:`.FeatureBase`]:
特征矩阵和生成的特征定义列表.
与dfs输出匹配.如果没有提供特征列表作为输入,则不会返回特征列表.为了获得一致的结果,请不要更改dfs输出的特征顺序.
"""
if feature_matrix.ww.schema is None:
feature_matrix.ww.init()
if pct_corr_threshold < 0 or pct_corr_threshold > 1:
raise ValueError(
"pct_corr_threshold must be a float between 0 and 1, inclusive.",
)
if features_to_check is None:
features_to_check = list(feature_matrix.columns)
else:
for f_name in features_to_check:
assert (
f_name in feature_matrix.columns
), "feature named {} is not in feature matrix".format(f_name)
if features_to_keep is None:
features_to_keep = []
to_select = ["numeric", Boolean, BooleanNullable]
fm = feature_matrix.ww[features_to_check]
fm_to_check = fm.ww.select(include=to_select)
dropped = set()
columns_to_check = fm_to_check.columns
# When two features are found to be highly correlated,
# we drop the more complex feature
# Columns produced later in dfs are more complex
for i in range(len(columns_to_check) - 1, 0, -1):
more_complex_name = columns_to_check[i]
more_complex_col = fm_to_check[more_complex_name]
# Convert boolean or Int64 column to be float64
if pd.api.types.is_bool_dtype(more_complex_col) or isinstance(
more_complex_col.dtype,
pd.Int64Dtype,
):
more_complex_col = more_complex_col.astype("float64")
for j in range(i - 1, -1, -1):
less_complex_name = columns_to_check[j]
less_complex_col = fm_to_check[less_complex_name]
# Convert boolean or Int64 column to be float64
if pd.api.types.is_bool_dtype(less_complex_col) or isinstance(
less_complex_col.dtype,
pd.Int64Dtype,
):
less_complex_col = less_complex_col.astype("float64")
if abs(more_complex_col.corr(less_complex_col)) >= pct_corr_threshold:
dropped.add(more_complex_name)
break
keep = [
f_name
for f_name in feature_matrix.columns
if (f_name in features_to_keep or f_name not in dropped)
]
return _apply_feature_selection(keep, feature_matrix, features)
def _apply_feature_selection(keep, feature_matrix, features=None):
new_matrix = feature_matrix[keep]
new_feature_names = set(new_matrix.columns)
if features is not None:
new_features = []
for f in features:
if f.number_output_features > 1:
slices = [
f[i]
for i in range(f.number_output_features)
if f[i].get_name() in new_feature_names
]
if len(slices) == f.number_output_features:
new_features.append(f)
else:
new_features.extend(slices)
else:
if f.get_name() in new_feature_names:
new_features.append(f)
return new_matrix, new_features
return new_matrix