Add feature_clustering_selection method

src/fklearn/tuning/model_agnostic_fc.py

@@ -3,6 +3,10 @@
 import pandas as pd
 from numpy import tril
 from toolz import curry
+import numpy as np
+from sklearn.linear_model import LinearRegression
+from sklearn.cluster import AgglomerativeClustering
+import itertools
 
 from fklearn.types import LogType
 
@@ -79,3 +83,167 @@ def variance_feature_selection(train_set: pd.DataFrame, features: List[str], thr
     return {"feature_var": feature_var.to_dict(),
             "features_to_drop": features_to_drop,
             "final_features": final_features}
+
+
+@curry
+def feature_clustering_selection(train_set: pd.DataFrame,
+                                 features: List[str],
+                                 dissimilarity_threshold: float = 0.5) -> LogType:
+    """
+    Feature selection based on feature clustering with absolute correlarion as distance metric.
+    One feature is selected from cluster, using the selection criteria of lower feature
+    "1 - R2 ratio". "1 - R2 ratio" = (1 - "own cluster R2") / (1 - "nearest cluster R2").
+    The higher is "own cluster R2", or the more the feature explain from the other features
+    from the cluster, the lower is "1 - R2 ratio". In parallel, the lower is "nearest cluster R2",
+    or the less the feature explain from the features from the nearest cluster, the lower is
+    "1 - R2 ratio".
+    The intuition behind this is to keep the most heterogenious information from the dataset,
+    keeping features that most represent the information present in the other features from its
+    cluster, and that is not already explained by the features from the nearest cluster.
+
+    Parameters
+    ----------
+    train_set : pd.DataFrame
+        A Pandas' DataFrame with the training data
+
+    features : list of str
+        The list of features to consider when dropping with correlation
+
+    dissimilarity_threshold : float
+        The dissimilarity (1 - absolute correlation) threshold. It will only cluster features
+        in which the dissimilarity were equal or under this threshold. Or, intuitively, will only
+        cluster features in which absolute correlation were equal or above (1 - this threshold).
+
+    Returns
+    ----------
+    log with feature scores ("own cluster R", "nearest cluster R2" and "1 - R2 ratio"), features to
+    drop and final features
+    """
+
+    # correlation matrix
+    corr_matrix = train_set[features].corr()
+
+    # dissimilarity matrix
+    dissimilarity_matrix = 1 - np.abs(corr_matrix)
+
+    # feature clustering
+    clustering = AgglomerativeClustering(
+        affinity='precomputed',
+        linkage='average',
+        n_clusters=None,
+        distance_threshold=dissimilarity_threshold)
+    clustering.fit(dissimilarity_matrix)
+
+    # unique labels
+    unique_labels = np.sort(np.unique(clustering.labels_))
+
+    unique_cluster_features = [(
+        cluster,
+        list(
+            dissimilarity_matrix.columns[np.where(clustering.labels_ == cluster)])) for cluster in unique_labels]
+
+    features_cluster_regression_data = [(
+        own_cluster_feature,
+        cluster,
+        tuple(
+            own_cluster_other_feature for own_cluster_other_feature
+            in own_cluster_features if own_cluster_feature != own_cluster_other_feature
+        )) for cluster, own_cluster_features in unique_cluster_features
+        for own_cluster_feature in own_cluster_features]
+
+    # feature scores
+    features_r2_scores = [(
+        f[0],
+        f[1],
+        0 if len(f[2]) == 0 else LinearRegression().fit(
+            train_set[features][list(f[2])],
+            train_set[features][f[0]]).score(
+                train_set[features][list(f[2])],
+                train_set[features][f[0]])) for f in features_cluster_regression_data]
+
+    df_features_ratio_scores = pd.DataFrame([(
+        f,
+        cluster,
+        own_cluster_score) for f, cluster, own_cluster_score in features_r2_scores],
+        columns=[
+            'feature',
+            'cluster',
+            'own_cluster_R2'])
+
+    # cluster pairs mean distances
+    cluster_pairs = [
+        element for element in itertools.product(*[unique_labels, unique_labels]) if element[0] < element[1]]
+    pairs = [pair for pair in cluster_pairs]
+    distances = [(
+        pair[0],
+        pair[1],
+        np.mean(dissimilarity_matrix.loc[
+            dissimilarity_matrix.columns[
+                np.where(clustering.labels_ == pair[0])[0]]][dissimilarity_matrix.columns[
+                    np.where(clustering.labels_ == pair[1])[0]]].values)) for pair in pairs]
+
+    # A->B and B->A cluster distances
+    distances = distances + [(distance[1], distance[0], distance[2]) for distance in distances]
+    df_clusters_distances = pd.DataFrame(
+        distances, columns=['cluster', 'neighbor', 'dissimilarity'])
+
+    # nearest cluster
+    df_nearest_cluster_pairs = df_clusters_distances.sort_values('dissimilarity').groupby('cluster').head(1)
+
+    # cluster pairs feature list
+    nearest_cluster_pairs_features = [(
+        row['cluster'],
+        list(dissimilarity_matrix.columns[
+            np.where(clustering.labels_ == row['cluster'])]),
+        row['neighbor'],
+        list(dissimilarity_matrix.columns[
+            np.where(clustering.labels_ == row['neighbor'])])) for ix, row in df_nearest_cluster_pairs.iterrows()]
+
+    features_nearest_cluster_regression_data = [(
+        own_cluster_feature,
+        own_cluster,
+        nearest_cluster,
+        nearest_cluster_features) for own_cluster, own_cluster_features, nearest_cluster, nearest_cluster_features
+        in nearest_cluster_pairs_features for own_cluster_feature in own_cluster_features]
+
+    # feature scores
+    dict_nearest_cluster_r2_scores = {
+        f[0]: {
+            'nearest_cluster': f[2],
+            'nearest_cluster_R2': LinearRegression().fit(
+                train_set[features][list(f[3])],
+                train_set[features][f[0]]).score(
+                    train_set[features][list(f[3])],
+                    train_set[features][f[0]])} for f in features_nearest_cluster_regression_data}
+
+    # nearest cluster data
+    df_features_ratio_scores['nearest_cluster'] = df_features_ratio_scores['feature'].apply(
+        lambda x: None if x not in dict_nearest_cluster_r2_scores
+        else dict_nearest_cluster_r2_scores[x]['nearest_cluster'])
+
+    df_features_ratio_scores['nearest_cluster_R2'] = df_features_ratio_scores['feature'].apply(
+        lambda x: None if x not in dict_nearest_cluster_r2_scores
+        else dict_nearest_cluster_r2_scores[x]['nearest_cluster_R2'])
+
+    # final scores (1-R2_ratio)
+    df_features_ratio_scores['1-R2_ratio'] = (
+        (1 - df_features_ratio_scores['own_cluster_R2']) / (1 - df_features_ratio_scores['nearest_cluster_R2']))
+
+    # best features
+    df_features_ratio_scores = df_features_ratio_scores.sort_values('cluster')
+    df_best_features = df_features_ratio_scores.sort_values(
+        ['1-R2_ratio', 'own_cluster_R2'],
+        ascending=[True, False]).groupby('cluster').head(1)
+
+    final_features = list(df_best_features['feature'].values)
+    features_to_drop = list(set(features) - set(final_features))
+
+    df_features_ratio_scores['is_selected'] = df_features_ratio_scores['feature'].apply(
+        lambda x: x in final_features)
+
+    return {
+        "df_feature_clustering": df_features_ratio_scores.sort_values(
+            ['cluster', '1-R2_ratio', 'own_cluster_R2'],
+            ascending=[True, True, False]).reset_index(drop=True).to_dict(),
+        "features_to_drop": features_to_drop,
+        "final_features": final_features}

tests/tuning/test_model_agnostic_fc.py

@@ -1,5 +1,6 @@
 import pandas as pd
-from fklearn.tuning.model_agnostic_fc import variance_feature_selection, correlation_feature_selection
+from fklearn.tuning.model_agnostic_fc import variance_feature_selection, \
+    correlation_feature_selection, feature_clustering_selection
 
 
 def test_correlation_feature_selection():
@@ -43,3 +44,52 @@ def test_variance_feature_selection():
 
     assert set(result["features_to_drop"]) == {'c'}
     assert set(result["final_features"]) == {'x', 'y', 'z', 'a', 'b'}
+
+
+def test_feature_clustering_selection():
+
+    train_set = pd.DataFrame({
+        'id': ["id1", "id2", "id3", "id4", "id5", "id6", "id7", "id8", "id9", "id10",
+               "id11", "id12", "id13", "id14", "id15", "id16", "id17", "id18", "id19", "id20"],
+        'x': [1.58, 2.82, 4.71, 2.2, 4.59, 7.03, 3.3, 4.64, 3.48, 5.31,
+              -1.29, 4.29, 6.86, 1.42, 6.47, 5.24, 7.46, 5.45, 5.71, 6.23],
+        'y': [4.5, -1.07, 0.61, 1.97, 2.96, -3.02, 3.37, -0.06, 1.73, -1.94,
+              8.01, 2.66, -0.08, 2.38, -0.68, -1.01, -5.9, 0.7, -0.76, -1.24],
+        'z': [11.93, 11.49, 10.09, 11.93, 10.15, 8.97, 10.55, 10.74, 11.12, 9.47,
+              13.91, 10.19, 8.73, 12.23, 9.3, 9.9, 8.63, 9.47, 9.81, 9.64],
+        'q': [40.57, 59.36, 36.24, 36.64, 126.97, 5.23, 38.58, 10.01, 105.32, 6.23,
+              11.27, 13.89, 21.11, 7.5, 165.71, 16.66, 12.35, 43.63, 39.77, 6.99],
+        'a': [48.96, 51.59, 49.2, 52.07, 51.83, 50.11, 51.28, 46.29, 52.45, 52.17,
+              50.64, 45.41, 45.44, 47.31, 52.72, 51.09, 49.56, 52.12, 48.99, 47.97],
+        'b': [6.89, 8.83, 6.68, 7.87, 8.11, 7.57, 9.0, 5.89, 7.75, 7.86,
+              8.94, 6.17, 3.13, 7.99, 8.76, 3.71, 7.73, 8.36, 6.45, 6.23]
+    })
+
+    features = ["x", "y", "z", "q", "b", "a"]
+
+    dissimilarity_threshold = 0.5
+    result = feature_clustering_selection(
+        train_set=train_set,
+        features=features,
+        dissimilarity_threshold=dissimilarity_threshold)
+
+    assert set(result["features_to_drop"]) == {'y', 'z', 'a'}
+    assert set(result["final_features"]) == {'x', 'b', 'q'}
+
+    dissimilarity_threshold = 0.0
+    result = feature_clustering_selection(
+        train_set=train_set,
+        features=features,
+        dissimilarity_threshold=dissimilarity_threshold)
+
+    assert set(result["features_to_drop"]) == set()
+    assert set(result["final_features"]) == {'q', 'a', 'b', 'y', 'z', 'x'}
+
+    dissimilarity_threshold = 1.0
+    result = feature_clustering_selection(
+        train_set=train_set,
+        features=features,
+        dissimilarity_threshold=dissimilarity_threshold)
+
+    assert set(result["features_to_drop"]) == {'a', 'y', 'q', 'b', 'z'}
+    assert set(result["final_features"]) == {'x'}