Feature importance

Author: yufongpeng

src/DecisionTree.jl

@@ -9,11 +9,12 @@ using Random
 using Statistics
 import AbstractTrees
 
-export Leaf, Node, Ensemble, print_tree, depth, build_stump, build_tree,
+export Leaf, Node, Root, Ensemble, print_tree, depth, build_stump, build_tree,
        prune_tree, apply_tree, apply_tree_proba, nfoldCV_tree, build_forest,
        apply_forest, apply_forest_proba, nfoldCV_forest, build_adaboost_stumps,
        apply_adaboost_stumps, apply_adaboost_stumps_proba, nfoldCV_stumps,
-       majority_vote, ConfusionMatrix, confusion_matrix, mean_squared_error, R2, load_data
+       majority_vote, ConfusionMatrix, confusion_matrix, mean_squared_error, R2, load_data,
+       impurity_importance, split_importance, permutation_importance, accuracy
 
 # ScikitLearn API
 export DecisionTreeClassifier, DecisionTreeRegressor, RandomForestClassifier,
@@ -42,17 +43,32 @@ end
 
 const LeafOrNode{S, T} = Union{Leaf{T}, Node{S, T}}
 
+struct Root{S, T}
+    node    :: LeafOrNode{S, T}
+    n_feat  :: Int
+    featim  :: Vector{Float64}   # impurity importance
+end
+
 struct Ensemble{S, T}
-    trees :: Vector{LeafOrNode{S, T}}
+    trees   :: Vector{LeafOrNode{S, T}}
+    n_feat  :: Int
+    featim  :: Vector{Float64}
 end
 
+
 is_leaf(l::Leaf) = true
 is_leaf(n::Node) = false
 
-zero(String) = ""
+zero(::Type{String}) = ""
 convert(::Type{Node{S, T}}, lf::Leaf{T}) where {S, T} = Node(0, zero(S), lf, Leaf(zero(T), [zero(T)]))
+convert(::Type{Root{S, T}}, node::LeafOrNode{S, T}) where {S, T} = Root{S, T}(node, 0, Float64[])
+convert(::Type{LeafOrNode{S, T}}, tree::Root{S, T}) where {S, T} = tree.node
 promote_rule(::Type{Node{S, T}}, ::Type{Leaf{T}}) where {S, T} = Node{S, T}
 promote_rule(::Type{Leaf{T}}, ::Type{Node{S, T}}) where {S, T} = Node{S, T}
+promote_rule(::Type{Root{S, T}}, ::Type{Leaf{T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Leaf{T}}, ::Type{Root{S, T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Root{S, T}}, ::Type{Node{S, T}}) where {S, T} = Root{S, T}
+promote_rule(::Type{Node{S, T}}, ::Type{Root{S, T}}) where {S, T} = Root{S, T}
 
 # make a Random Number Generator object
 mk_rng(rng::Random.AbstractRNG) = rng
@@ -75,10 +91,12 @@ include("abstract_trees.jl")
 
 length(leaf::Leaf) = 1
 length(tree::Node) = length(tree.left) + length(tree.right)
+length(tree::Root) = length(tree.node)
 length(ensemble::Ensemble) = length(ensemble.trees)
 
 depth(leaf::Leaf) = 0
 depth(tree::Node) = 1 + max(depth(tree.left), depth(tree.right))
+depth(tree::Root) = depth(tree.node)
 
 function print_tree(io::IO, leaf::Leaf, depth=-1, indent=0; feature_names=nothing)
     n_matches = count(leaf.values .== leaf.majority)
@@ -90,6 +108,13 @@ function print_tree(leaf::Leaf, depth=-1, indent=0; feature_names=nothing)
 end
 
 
+function print_tree(io::IO, tree::Root, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
+    return print_tree(io, tree.node, depth, indent; sigdigits=sigdigits, feature_names=feature_names)
+end
+function print_tree(tree::Root, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
+    return print_tree(stdout, tree, depth, indent; sigdigits=sigdigits, feature_names=feature_names)
+end
+
 """
     print_tree([io::IO,] tree::Node, depth=-1, indent=0; sigdigits=4, feature_names=nothing)
 
@@ -113,9 +138,9 @@ Feature 3 < -28.15 ?
    └─ 8 : 1227/3508
 ```
 
-To facilitate visualisation of trees using third party packages, a `DecisionTree.Leaf` object or
-`DecisionTree.Node` object can be wrapped to obtain a tree structure implementing the
-AbstractTrees.jl interface. See  [`wrap`](@ref)` for details.
+To facilitate visualisation of trees using third party packages, a `DecisionTree.Leaf` object,
+`DecisionTree.Node` object or  `DecisionTree.Root` object can be wrapped to obtain a tree structure implementing the 
+AbstractTrees.jl interface. See  [`wrap`](@ref)` for details. 
 """
 function print_tree(io::IO, tree::Node, depth=-1, indent=0; sigdigits=2, feature_names=nothing)
     if depth == indent
@@ -149,6 +174,12 @@ function show(io::IO, tree::Node)
     print(io,   "Depth:  $(depth(tree))")
 end
 
+function show(io::IO, tree::Root)
+    println(io, "Decision Tree")
+    println(io, "Leaves: $(length(tree))")
+    print(io,   "Depth:  $(depth(tree))")
+end
+
 function show(io::IO, ensemble::Ensemble)
     println(io, "Ensemble of Decision Trees")
     println(io, "Trees:      $(length(ensemble))")

src/abstract_trees.jl

@@ -66,6 +66,7 @@ In the first case `dc` gets just wrapped, no information is added. No. 2 adds fe
 as well as class labels. In the last two cases either of this information is added (Note the 
 trailing comma; it's needed to make it a tuple).
 """
+wrap(tree::DecisionTree.Root, info::NamedTuple = NamedTuple()) = wrap(tree.node, info)
 wrap(node::DecisionTree.Node, info::NamedTuple = NamedTuple()) = InfoNode(node, info)
 wrap(leaf::DecisionTree.Leaf, info::NamedTuple = NamedTuple()) = InfoLeaf(leaf, info)