Bootstrap Aggregation

algorithms/linfa-ensemble/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "linfa-ensemble"
+version = "0.6.0"
+edition = "2018"
+authors = ["James Knight <[email protected]>", "James Kay <[email protected]>"]
+description = "A general method for creating ensemble classifiers"
+license = "MIT/Apache-2.0"
+
+repository = "https://github.com/rust-ml/linfa"
+readme = "README.md"
+
+keywords = ["machine-learning", "linfa", "ensemble"]
+categories = ["algorithms", "mathematics", "science"]
+
+[features]
+default = []
+serde = ["serde_crate", "ndarray/serde"]
+
+[dependencies.serde_crate]
+package = "serde"
+optional = true
+version = "1.0"
+default-features = false
+features = ["std", "derive"]
+
+[dependencies]
+ndarray = { version = "0.15" , features = ["rayon", "approx"]}
+ndarray-rand = "0.14"
+rand = "0.8.5"
+
+linfa = { version = "0.6.0", path = "../.." }
+linfa-trees = { version = "0.6.0", path = "../linfa-trees"}
+
+[dev-dependencies]
+linfa-datasets = { version = "0.6.0", path = "../../datasets/", features = ["iris"] }
+

algorithms/linfa-ensemble/README.md

@@ -0,0 +1,21 @@
+# Enseble Learning
+
+`linfa-ensemble` provides pure Rust implementations of Ensemble Learning algorithms for the Linfa toolkit.
+
+## The Big Picture
+
+`linfa-ensemble` is a crate in the [`linfa`](https://crates.io/crates/linfa) ecosystem, an effort to create a toolkit for classical Machine Learning implemented in pure Rust, akin to Python's `scikit-learn`.
+
+## Current state
+
+`linfa-ensemble` currently provides an implementation of bootstrap aggregation (bagging) for other classifers provided in linfa.
+
+## Examples
+
+You can find examples in the `examples/` directory. To run an bootstrap aggregation for ensemble of decision trees (a Random Forest) use:
+
+```bash
+$ cargo run --example randomforest_iris --release
+```
+
+

algorithms/linfa-ensemble/examples/randomforest_iris.rs

@@ -0,0 +1,38 @@
+use linfa::prelude::{Fit, Predict, ToConfusionMatrix};
+use linfa_ensemble::{EnsembleLearnerParams};
+use linfa_trees::DecisionTree;
+use ndarray_rand::rand::SeedableRng;
+use rand::rngs::SmallRng;
+
+fn main() {
+    //Number of models in the ensemble
+    let ensemble_size = 100;
+    //Proportion of training data given to each model
+    let bootstrap_proportion = 0.7;
+
+    //Create ensemble learner
+    let mut learner = EnsembleLearnerParams::new();
+    learner
+        .ensemble_size(ensemble_size)
+        .bootstrap_proportion(bootstrap_proportion)
+        .model_params(DecisionTree::params());
+
+    //Load dataset
+    let mut rng = SmallRng::seed_from_u64(42);
+    let (train, test) = linfa_datasets::iris()
+        .shuffle(&mut rng)
+        .split_with_ratio(0.8);
+
+    //Train ensemble learner model
+    let model = learner.fit(&train).unwrap();
+
+    //Return highest ranking predictions
+    let final_predictions_ensemble = model.predict(&test);
+    println!("Final Predictions: \n{:?}", final_predictions_ensemble);
+
+    let cm = final_predictions_ensemble.confusion_matrix(&test).unwrap();
+
+    println!("{:?}", cm);
+    println!("Test accuracy: {} \n with default Decision Tree params, \n Ensemble Size: {},\n Bootstrap Proportion: {}",
+    100.0 * cm.accuracy(), ensemble_size, bootstrap_proportion);
+}

algorithms/linfa-ensemble/src/ensemble.rs

@@ -0,0 +1,157 @@
+use linfa::{
+    dataset::{AsTargets, AsTargetsMut, FromTargetArrayOwned, Records},
+    error::{Error},
+    traits::*,
+    DatasetBase,
+};
+use ndarray::{
+    Array2, Axis, Array, Dimension
+};
+use std::{
+    cmp::Eq,
+    collections::HashMap,
+    hash::Hash,
+};
+use rand::Rng;
+use rand::rngs::ThreadRng;
+
+pub struct EnsembleLearner<M> {
+    pub models: Vec<M>,
+}
+
+impl<M> EnsembleLearner<M> {
+
+    // Generates prediction iterator returning predictions from each model
+    pub fn generate_predictions<'b, R: Records, T>(&'b self, x: &'b R) -> impl Iterator<Item = T> + 'b
+    where M: Predict<&'b R, T> {
+        self.models.iter().map(move |m| m.predict(x))
+    }
+
+    // Consumes prediction iterator to return all predictions
+    pub fn aggregate_predictions<Ys: Iterator>(&self, ys: Ys)
+    -> impl Iterator<Item = Vec<(Array<<Ys::Item as AsTargets>::Elem, <<Ys::Item as AsTargets>::Ix as Dimension>::Smaller >, usize)>>
+    where
+        Ys::Item: AsTargets,
+        <Ys::Item as AsTargets>::Elem: Copy + Eq + Hash,
+    {
+        let mut prediction_maps = Vec::new();
+
+        for y in ys {
+            let targets = y.as_targets();
+            let no_targets = targets.shape()[0];
+
+            for i in 0..no_targets {
+                if prediction_maps.len() == i {
+                    prediction_maps.push(HashMap::new());
+                }
+                *prediction_maps[i].entry(y.as_targets().index_axis(Axis(0), i).to_owned()).or_insert(0) += 1;
+            }
+        }
+
+        prediction_maps.into_iter().map(|xs| {
+            let mut xs: Vec<_> = xs.into_iter().collect();
+            xs.sort_by(|(_, x), (_, y)| y.cmp(x));
+            xs
+        })
+    }
+
+}
+
+impl<F: Clone, T, M>
+PredictInplace<Array2<F>, T> for EnsembleLearner<M>
+where
+    M: PredictInplace<Array2<F>, T>,
+    <T as AsTargets>::Elem: Copy + Eq + Hash,
+    T: AsTargets + AsTargetsMut<Elem = <T as AsTargets>::Elem>,
+{
+    fn predict_inplace(&self, x: &Array2<F>, y: &mut T) {
+        let mut y_array = y.as_targets_mut();
+        assert_eq!(
+            x.nrows(),
+            y_array.len_of(Axis(0)),
+            "The number of data points must match the number of outputs."
+        );
+
+        let mut predictions = self.generate_predictions(x);
+        let aggregated_predictions = self.aggregate_predictions(&mut predictions);
+
+        for (target, output) in y_array.axis_iter_mut(Axis(0)).zip(aggregated_predictions.into_iter()) {
+            for (t, o) in target.into_iter().zip(output[0].0.iter()) {
+                *t = *o;
+            }
+        }
+    }
+
+    fn default_target(&self, x: &Array2<F>) -> T {
+        self.models[0].default_target(x)
+    }
+}
+
+pub struct EnsembleLearnerParams<P, R: Rng + Clone> {
+    pub ensemble_size: usize,
+    pub bootstrap_proportion: f64,
+    pub model_params: P,
+    pub rng: R
+}
+
+impl<P> EnsembleLearnerParams<P, ThreadRng> {
+    pub fn new(model_params: P) -> EnsembleLearnerParams<P, ThreadRng> {
+        return Self::new_fixed_rng(model_params, rand::thread_rng())
+    }
+}
+
+impl<P, R: Rng + Clone> EnsembleLearnerParams<P, R> {
+    pub fn new_fixed_rng(model_params: P, rng: R) -> EnsembleLearnerParams<P, R> {
+        EnsembleLearnerParams {
+            ensemble_size: 1,
+            bootstrap_proportion: 1.0,
+            model_params: model_params,
+            rng: rng
+        }
+    }
+
+    pub fn ensemble_size(&mut self, size: usize) -> &mut EnsembleLearnerParams<P, R> {
+        assert!(size > 0, "ensemble_size cannot be less than 1. Ensembles must consist of at least one model.");
+        self.ensemble_size = size;
+        self
+    }
+
+    pub fn bootstrap_proportion(&mut self, proportion: f64) -> &mut EnsembleLearnerParams<P, R> {
+        assert!(proportion > 0.0, "bootstrap_proportion must be greater than 0. Must provide some data to each model.");
+        self.bootstrap_proportion = proportion;
+        self
+    }
+
+}
+
+impl<D, T, P: Fit<Array2<D>, T::Owned, Error>, R: Rng + Clone>
+     Fit<Array2<D>, T, Error> for EnsembleLearnerParams<P, R>
+where
+    D: Clone,
+    T: FromTargetArrayOwned,
+    T::Elem: Copy + Eq + Hash,
+    T::Owned: AsTargets,
+{
+    type Object = EnsembleLearner<P::Object>;
+
+    fn fit(&self, dataset: &DatasetBase<Array2<D>, T>) -> Result<Self::Object, Error> {
+
+        let mut models = Vec::new();
+        let mut rng = self.rng.clone();
+
+        let dataset_size = ((dataset.records.shape()[0] as f64) * self.bootstrap_proportion).ceil() as usize;
+
+        let iter = dataset.bootstrap_samples(dataset_size, &mut rng);
+
+        for train in iter {
+            let model = self.model_params.fit(&train).unwrap();
+            models.push(model);
+
+            if models.len() == self.ensemble_size {
+                break
+            }
+        }
+
+        Ok(EnsembleLearner { models })
+    }
+}

algorithms/linfa-ensemble/src/lib.rs

@@ -0,0 +1,3 @@
+mod ensemble;
+
+pub use ensemble::*;

src/dataset/impl_dataset.rs

@@ -2,7 +2,7 @@ use super::{
     super::traits::{Predict, PredictInplace},
     iter::{ChunksIter, DatasetIter, Iter},
     AsSingleTargets, AsTargets, AsTargetsMut, CountedTargets, Dataset, DatasetBase, DatasetView,
-    Float, FromTargetArray, Label, Labels, Records, Result, TargetDim,
+    Float, FromTargetArray, FromTargetArrayOwned, Label, Labels, Records, Result, TargetDim,
 };
 use crate::traits::Fit;
 use ndarray::{concatenate, prelude::*, Data, DataMut, Dimension};
@@ -418,7 +418,7 @@ where
 impl<'b, F: Clone, E: Copy + 'b, D, T> DatasetBase<ArrayBase<D, Ix2>, T>
 where
     D: Data<Elem = F>,
-    T: FromTargetArray<'b, Elem = E>,
+    T: FromTargetArrayOwned<Elem = E>,
     T::Owned: AsTargets,
 {
     /// Apply bootstrapping for samples and features
@@ -441,7 +441,7 @@ where
         &'b self,
         sample_feature_size: (usize, usize),
         rng: &'b mut R,
-    ) -> impl Iterator<Item = DatasetBase<Array2<F>, <T as FromTargetArray<'b>>::Owned>> + 'b {
+    ) -> impl Iterator<Item = DatasetBase<Array2<F>, T::Owned>> + 'b {
         std::iter::repeat(()).map(move |_| {
             // sample with replacement
             let indices = (0..sample_feature_size.0)
@@ -481,7 +481,7 @@ where
         &'b self,
         num_samples: usize,
         rng: &'b mut R,
-    ) -> impl Iterator<Item = DatasetBase<Array2<F>, <T as FromTargetArray<'b>>::Owned>> + 'b {
+    ) -> impl Iterator<Item = DatasetBase<Array2<F>, T::Owned>> + 'b {
         std::iter::repeat(()).map(move |_| {
             // sample with replacement
             let indices = (0..num_samples)
@@ -515,7 +515,7 @@ where
         &'b self,
         num_features: usize,
         rng: &'b mut R,
-    ) -> impl Iterator<Item = DatasetBase<Array2<F>, <T as FromTargetArray<'b>>::Owned>> + 'b {
+    ) -> impl Iterator<Item = DatasetBase<Array2<F>, T::Owned>> + 'b {
         std::iter::repeat(()).map(move |_| {
             let targets = T::new_targets(self.as_targets().to_owned());
 

src/dataset/impl_targets.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use super::{
     AsMultiTargets, AsMultiTargetsMut, AsProbabilities, AsSingleTargets, AsSingleTargetsMut,
-    AsTargets, AsTargetsMut, CountedTargets, DatasetBase, FromTargetArray, Label, Labels, Pr,
+    AsTargets, AsTargetsMut, CountedTargets, DatasetBase, FromTargetArray, FromTargetArrayOwned, Label, Labels, Pr,
     TargetDim,
 };
 use ndarray::{
@@ -25,21 +25,25 @@ impl<'a, L, S: Data<Elem = L>, I: TargetDim> AsTargets for ArrayBase<S, I> {
 impl<T: AsTargets<Ix = Ix1>> AsSingleTargets for T {}
 impl<T: AsTargets<Ix = Ix2>> AsMultiTargets for T {}
 
-impl<'a, L: Clone + 'a, S: Data<Elem = L>, I: TargetDim> FromTargetArray<'a> for ArrayBase<S, I> {
+impl<'a, L: Clone + 'a, S: Data<Elem = L>, I: TargetDim> FromTargetArrayOwned for ArrayBase<S, I> {
     type Owned = ArrayBase<OwnedRepr<L>, I>;
-    type View = ArrayBase<ViewRepr<&'a L>, I>;
 
     /// Returns an owned representation of the target array
     fn new_targets(targets: Array<L, I>) -> Self::Owned {
         targets
     }
+}
+
+impl<'a, L: Clone + 'a, S: Data<Elem = L>, I: TargetDim> FromTargetArray<'a> for ArrayBase<S, I> {
+    type View = ArrayBase<ViewRepr<&'a L>, I>;
 
     /// Returns a reference to the target array
     fn new_targets_view(targets: ArrayView<'a, L, I>) -> Self::View {
         targets
     }
 }
 
+
 impl<L, S: DataMut<Elem = L>, I: TargetDim> AsTargetsMut for ArrayBase<S, I> {
     type Elem = L;
     type Ix = I;
@@ -79,23 +83,29 @@ impl<L: Label, T: AsTargetsMut<Elem = L>> AsTargetsMut for CountedTargets<L, T>
     }
 }
 
-impl<'a, L: Label + 'a, T> FromTargetArray<'a> for CountedTargets<L, T>
+impl<L: Label, T> FromTargetArrayOwned for CountedTargets<L, T>
 where
-    T: FromTargetArray<'a, Elem = L>,
+    T: FromTargetArrayOwned<Elem = L>,
     T::Owned: Labels<Elem = L>,
-    T::View: Labels<Elem = L>,
 {
     type Owned = CountedTargets<L, T::Owned>;
-    type View = CountedTargets<L, T::View>;
 
     fn new_targets(targets: Array<L, T::Ix>) -> Self::Owned {
         let targets = T::new_targets(targets);
-
         CountedTargets {
             labels: targets.label_count(),
             targets,
         }
     }
+}
+
+
+impl<'a, L: Label + 'a, T> FromTargetArray<'a> for CountedTargets<L, T>
+where
+    T: FromTargetArray<'a, Elem = L>,
+    T::View: Labels<Elem = L>,
+{
+    type View = CountedTargets<L, T::View>;
 
     fn new_targets_view(targets: ArrayView<'a, L, T::Ix>) -> Self::View {
         let targets = T::new_targets_view(targets);