diff --git a/src/fklearn/tuning/model_agnostic_fc.py b/src/fklearn/tuning/model_agnostic_fc.py index da292444..61258e2b 100644 --- a/src/fklearn/tuning/model_agnostic_fc.py +++ b/src/fklearn/tuning/model_agnostic_fc.py @@ -3,6 +3,10 @@ import pandas as pd from numpy import tril from toolz import curry +import numpy as np +from sklearn.linear_model import LinearRegression +from sklearn.cluster import AgglomerativeClustering +import itertools from fklearn.types import LogType @@ -79,3 +83,167 @@ def variance_feature_selection(train_set: pd.DataFrame, features: List[str], thr return {"feature_var": feature_var.to_dict(), "features_to_drop": features_to_drop, "final_features": final_features} + + +@curry +def feature_clustering_selection(train_set: pd.DataFrame, + features: List[str], + dissimilarity_threshold: float = 0.5) -> LogType: + """ + Feature selection based on feature clustering with absolute correlarion as distance metric. + One feature is selected from cluster, using the selection criteria of lower feature + "1 - R2 ratio". "1 - R2 ratio" = (1 - "own cluster R2") / (1 - "nearest cluster R2"). + The higher is "own cluster R2", or the more the feature explain from the other features + from the cluster, the lower is "1 - R2 ratio". In parallel, the lower is "nearest cluster R2", + or the less the feature explain from the features from the nearest cluster, the lower is + "1 - R2 ratio". + The intuition behind this is to keep the most heterogenious information from the dataset, + keeping features that most represent the information present in the other features from its + cluster, and that is not already explained by the features from the nearest cluster. + + Parameters + ---------- + train_set : pd.DataFrame + A Pandas' DataFrame with the training data + + features : list of str + The list of features to consider when dropping with correlation + + dissimilarity_threshold : float + The dissimilarity (1 - absolute correlation) threshold. It will only cluster features + in which the dissimilarity were equal or under this threshold. Or, intuitively, will only + cluster features in which absolute correlation were equal or above (1 - this threshold). + + Returns + ---------- + log with feature scores ("own cluster R", "nearest cluster R2" and "1 - R2 ratio"), features to + drop and final features + """ + + # correlation matrix + corr_matrix = train_set[features].corr() + + # dissimilarity matrix + dissimilarity_matrix = 1 - np.abs(corr_matrix) + + # feature clustering + clustering = AgglomerativeClustering( + affinity='precomputed', + linkage='average', + n_clusters=None, + distance_threshold=dissimilarity_threshold) + clustering.fit(dissimilarity_matrix) + + # unique labels + unique_labels = np.sort(np.unique(clustering.labels_)) + + unique_cluster_features = [( + cluster, + list( + dissimilarity_matrix.columns[np.where(clustering.labels_ == cluster)])) for cluster in unique_labels] + + features_cluster_regression_data = [( + own_cluster_feature, + cluster, + tuple( + own_cluster_other_feature for own_cluster_other_feature + in own_cluster_features if own_cluster_feature != own_cluster_other_feature + )) for cluster, own_cluster_features in unique_cluster_features + for own_cluster_feature in own_cluster_features] + + # feature scores + features_r2_scores = [( + f[0], + f[1], + 0 if len(f[2]) == 0 else LinearRegression().fit( + train_set[features][list(f[2])], + train_set[features][f[0]]).score( + train_set[features][list(f[2])], + train_set[features][f[0]])) for f in features_cluster_regression_data] + + df_features_ratio_scores = pd.DataFrame([( + f, + cluster, + own_cluster_score) for f, cluster, own_cluster_score in features_r2_scores], + columns=[ + 'feature', + 'cluster', + 'own_cluster_R2']) + + # cluster pairs mean distances + cluster_pairs = [ + element for element in itertools.product(*[unique_labels, unique_labels]) if element[0] < element[1]] + pairs = [pair for pair in cluster_pairs] + distances = [( + pair[0], + pair[1], + np.mean(dissimilarity_matrix.loc[ + dissimilarity_matrix.columns[ + np.where(clustering.labels_ == pair[0])[0]]][dissimilarity_matrix.columns[ + np.where(clustering.labels_ == pair[1])[0]]].values)) for pair in pairs] + + # A->B and B->A cluster distances + distances = distances + [(distance[1], distance[0], distance[2]) for distance in distances] + df_clusters_distances = pd.DataFrame( + distances, columns=['cluster', 'neighbor', 'dissimilarity']) + + # nearest cluster + df_nearest_cluster_pairs = df_clusters_distances.sort_values('dissimilarity').groupby('cluster').head(1) + + # cluster pairs feature list + nearest_cluster_pairs_features = [( + row['cluster'], + list(dissimilarity_matrix.columns[ + np.where(clustering.labels_ == row['cluster'])]), + row['neighbor'], + list(dissimilarity_matrix.columns[ + np.where(clustering.labels_ == row['neighbor'])])) for ix, row in df_nearest_cluster_pairs.iterrows()] + + features_nearest_cluster_regression_data = [( + own_cluster_feature, + own_cluster, + nearest_cluster, + nearest_cluster_features) for own_cluster, own_cluster_features, nearest_cluster, nearest_cluster_features + in nearest_cluster_pairs_features for own_cluster_feature in own_cluster_features] + + # feature scores + dict_nearest_cluster_r2_scores = { + f[0]: { + 'nearest_cluster': f[2], + 'nearest_cluster_R2': LinearRegression().fit( + train_set[features][list(f[3])], + train_set[features][f[0]]).score( + train_set[features][list(f[3])], + train_set[features][f[0]])} for f in features_nearest_cluster_regression_data} + + # nearest cluster data + df_features_ratio_scores['nearest_cluster'] = df_features_ratio_scores['feature'].apply( + lambda x: None if x not in dict_nearest_cluster_r2_scores + else dict_nearest_cluster_r2_scores[x]['nearest_cluster']) + + df_features_ratio_scores['nearest_cluster_R2'] = df_features_ratio_scores['feature'].apply( + lambda x: None if x not in dict_nearest_cluster_r2_scores + else dict_nearest_cluster_r2_scores[x]['nearest_cluster_R2']) + + # final scores (1-R2_ratio) + df_features_ratio_scores['1-R2_ratio'] = ( + (1 - df_features_ratio_scores['own_cluster_R2']) / (1 - df_features_ratio_scores['nearest_cluster_R2'])) + + # best features + df_features_ratio_scores = df_features_ratio_scores.sort_values('cluster') + df_best_features = df_features_ratio_scores.sort_values( + ['1-R2_ratio', 'own_cluster_R2'], + ascending=[True, False]).groupby('cluster').head(1) + + final_features = list(df_best_features['feature'].values) + features_to_drop = list(set(features) - set(final_features)) + + df_features_ratio_scores['is_selected'] = df_features_ratio_scores['feature'].apply( + lambda x: x in final_features) + + return { + "df_feature_clustering": df_features_ratio_scores.sort_values( + ['cluster', '1-R2_ratio', 'own_cluster_R2'], + ascending=[True, True, False]).reset_index(drop=True).to_dict(), + "features_to_drop": features_to_drop, + "final_features": final_features} diff --git a/tests/tuning/test_model_agnostic_fc.py b/tests/tuning/test_model_agnostic_fc.py index 90c26641..b7715da7 100644 --- a/tests/tuning/test_model_agnostic_fc.py +++ b/tests/tuning/test_model_agnostic_fc.py @@ -1,5 +1,6 @@ import pandas as pd -from fklearn.tuning.model_agnostic_fc import variance_feature_selection, correlation_feature_selection +from fklearn.tuning.model_agnostic_fc import variance_feature_selection, \ + correlation_feature_selection, feature_clustering_selection def test_correlation_feature_selection(): @@ -43,3 +44,52 @@ def test_variance_feature_selection(): assert set(result["features_to_drop"]) == {'c'} assert set(result["final_features"]) == {'x', 'y', 'z', 'a', 'b'} + + +def test_feature_clustering_selection(): + + train_set = pd.DataFrame({ + 'id': ["id1", "id2", "id3", "id4", "id5", "id6", "id7", "id8", "id9", "id10", + "id11", "id12", "id13", "id14", "id15", "id16", "id17", "id18", "id19", "id20"], + 'x': [1.58, 2.82, 4.71, 2.2, 4.59, 7.03, 3.3, 4.64, 3.48, 5.31, + -1.29, 4.29, 6.86, 1.42, 6.47, 5.24, 7.46, 5.45, 5.71, 6.23], + 'y': [4.5, -1.07, 0.61, 1.97, 2.96, -3.02, 3.37, -0.06, 1.73, -1.94, + 8.01, 2.66, -0.08, 2.38, -0.68, -1.01, -5.9, 0.7, -0.76, -1.24], + 'z': [11.93, 11.49, 10.09, 11.93, 10.15, 8.97, 10.55, 10.74, 11.12, 9.47, + 13.91, 10.19, 8.73, 12.23, 9.3, 9.9, 8.63, 9.47, 9.81, 9.64], + 'q': [40.57, 59.36, 36.24, 36.64, 126.97, 5.23, 38.58, 10.01, 105.32, 6.23, + 11.27, 13.89, 21.11, 7.5, 165.71, 16.66, 12.35, 43.63, 39.77, 6.99], + 'a': [48.96, 51.59, 49.2, 52.07, 51.83, 50.11, 51.28, 46.29, 52.45, 52.17, + 50.64, 45.41, 45.44, 47.31, 52.72, 51.09, 49.56, 52.12, 48.99, 47.97], + 'b': [6.89, 8.83, 6.68, 7.87, 8.11, 7.57, 9.0, 5.89, 7.75, 7.86, + 8.94, 6.17, 3.13, 7.99, 8.76, 3.71, 7.73, 8.36, 6.45, 6.23] + }) + + features = ["x", "y", "z", "q", "b", "a"] + + dissimilarity_threshold = 0.5 + result = feature_clustering_selection( + train_set=train_set, + features=features, + dissimilarity_threshold=dissimilarity_threshold) + + assert set(result["features_to_drop"]) == {'y', 'z', 'a'} + assert set(result["final_features"]) == {'x', 'b', 'q'} + + dissimilarity_threshold = 0.0 + result = feature_clustering_selection( + train_set=train_set, + features=features, + dissimilarity_threshold=dissimilarity_threshold) + + assert set(result["features_to_drop"]) == set() + assert set(result["final_features"]) == {'q', 'a', 'b', 'y', 'z', 'x'} + + dissimilarity_threshold = 1.0 + result = feature_clustering_selection( + train_set=train_set, + features=features, + dissimilarity_threshold=dissimilarity_threshold) + + assert set(result["features_to_drop"]) == {'a', 'y', 'q', 'b', 'z'} + assert set(result["final_features"]) == {'x'}