Computing Feature Importance with OneHotEncoded Features - scikit-learn

Question

Is it possible to compute feature importance (with Random Forest) in scikit learn when features have been onehotencoded?

Answer 1

Here's an example of how to combine feature names with their importances:
from sklearn.feature_extraction import DictVectorizer
from sklearn.preprocessing import FunctionTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
# some example data
X = pd.DataFrame({'feature': ['value1', 'value2', 'value2', 'value1', 'value2']})
y = [1, 0, 0, 1, 1]
# translate rows to dicts
def row_to_dict(X, y=None):
return X.apply(dict, axis=1)
# define prediction model
ft = FunctionTransformer(row_to_dict, validate=False)
dv = DictVectorizer()
rf = RandomForestClassifier()
# glue steps together
model = make_pipeline(ft, dv, rf)
# train
model.fit(X, y)
# get feature importances
feature_importances = zip(dv.feature_names_, rf.feature_importances_)
# have a look
print feature_importances

Answer 2

Assuming that you have a pipeline with
a 'pre' step where you implement OneHotEncoder,
a 'clf' step where you define the classifier
the key of the categorical transformation is given as 'cat'
The following function will combine the feature importance of categorical features.
import numpy as np
import pandas as pd
import imblearn
def compute_feature_importance(model):
"""
Create feature importance using sklearn's ensemble models model.feature_importances_ property.
Parameters
----------
model : estimator instance (either sklearn.Pipeline, imblearn.Pipeline or a classifier)
PRE-FITTED classifier or a PRE-FITTED Pipeline in which the last estimator is a classifier.
Returns
-------
fi_df : Pandas DataFrame with feature_names and feature_importance
"""
if type(model) == imblearn.pipeline.Pipeline:
# If the user is using a pipeline model,
# the importance of the feature is calculated in this if block!
pre_model = model['pre'] # Pre step of the pipeline
classifier = model['clf'] # Classifier of the pipeline
ct = model.named_steps['pre'] # Define the column transform for the given pipeline model
# The following line will get the feature names.
feature_names = pre_model.get_feature_names_out()
feature_importance = np.array(classifier.feature_importances_)
# Create a DataFrame using a Dictionary
data = {'feature_names': feature_names, 'feature_importance': feature_importance}
fi_df = pd.DataFrame(data)
# Sort the DataFrame in order decreasing feature importance
fi_df.sort_values(by=['feature_importance'], ascending=False, inplace=True)
if 'cat' in ct.named_transformers_.keys() and hasattr(ct.named_transformers_['cat'], 'feature_names_in_'):
# We first have to apply a column transform and then sum up the feature importance values of individual OneHotEncoder columns.
# Original categorical features list. Categorical features before applying OneHotEncoder
original_cat_features = ct.named_transformers_['cat'].feature_names_in_.tolist()
# Categorical features list after applying OneHotEncoder
all_cat_list = ct.named_transformers_['cat'].get_feature_names_out(original_cat_features).tolist()
# A for loop for original_cat_features to find the one hot encoded features corresponding to each original categorical feature
for original_cat_feature in original_cat_features:
# List of one hot encoded features corresponding to each original categorical feature
cat_list = [i for i in all_cat_list if i.startswith(original_cat_feature)]
# OneHotEncoded columns must be renamed.
# ct.named transformers['cat'].get_feature_names_out(original cat_features) returns column names missing "cat__" in front.
# Let's fix it easily!
for i, element in enumerate(cat_list):
cat_list[i] = 'cat__' + element
# Slice fi_df dataframe to return the only rows for the associated OneHotEncoded features names (cat_list)
# and then sum the feature importance values
cat_sum = fi_df[fi_df['feature_names'].isin(cat_list)]['feature_importance'].sum()
# Slice fi_df dataframe to return the only rows other than categorical features.
# In other words, dataframe with numerical features
fi_df = fi_df[~fi_df['feature_names'].isin(cat_list)]
# Create a temporary dictionary to return the originial categorical feature
# and the summation of OneHotEncoded features importances
temp_dict = {'feature_names': original_cat_feature, 'feature_importance': cat_sum}
# Append the temporary_dict to the dataframe
fi_df = fi_df.append(temp_dict, ignore_index=True)
# Sort the DataFrame in order decreasing feature importance
fi_df.sort_values(by=['feature_importance'], ascending=False, inplace=True)
# Getting rid of the prefixes of the feature names
prefixes = ('num__', 'cat__', 'remainder__', 'scaler__')
for prefix in prefixes:
fi_df['feature_names'] = fi_df['feature_names'].apply(lambda x: str(x).replace(prefix,""))
return fi_df