Making more changes to incorporate an outlier removal step along with different regression implementations

Author: ooples

src/Enums/MatrixDecomposition.cs

@@ -20,5 +20,6 @@ public enum MatrixDecomposition
     GramSchmidt,
     Lu,
     Qr,
-    Svd
+    Svd,
+    Normal
 }

src/Enums/RegularizationType.cs

@@ -0,0 +1,9 @@
+namespace AiDotNet.Enums;
+
+public enum RegularizationType
+{
+    None,
+    L1,
+    L2,
+    ElasticNet
+}

src/Extensions/MatrixExtensions.cs

@@ -0,0 +1,19 @@
+namespace AiDotNet.Extensions;
+
+public static class MatrixExtensions
+{
+    public static Matrix<T> AddConstantColumn<T>(this Matrix<T> matrix, T value)
+    {
+        var newMatrix = new Matrix<T>(matrix.Rows, matrix.Columns + 1);
+        for (int i = 0; i < matrix.Rows; i++)
+        {
+            newMatrix[i, 0] = value;
+            for (int j = 0; j < matrix.Columns; j++)
+            {
+                newMatrix[i, j + 1] = matrix[i, j];
+            }
+        }
+
+        return newMatrix;
+    }
+}

src/Extensions/VectorExtensions.cs

@@ -2,4 +2,14 @@
 
 public static class VectorExtensions
 {
+    public static Vector<T> Slice<T>(this Vector<T> vector, int start, int length)
+    {
+        var slicedVector = new Vector<T>(length);
+        for (int i = 0; i < length; i++)
+        {
+            slicedVector[i] = vector[start + i];
+        }
+
+        return slicedVector;
+    }
 }

src/Factories/NormalizerFactory.cs

@@ -1,6 +1,4 @@
-using AiDotNet.Normalizers;
-
-namespace AiDotNet.Factories;
+namespace AiDotNet.Factories;
 
 public class NormalizerFactory<T>
 {

src/Factories/RegressionFactory.cs

@@ -0,0 +1,21 @@
+global using AiDotNet.Regression;
+
+namespace AiDotNet.Factories;
+
+public static class RegressionFactory
+{
+    public static MultivariateRegression<T> CreateRidgeRegression<T>(INumericOperations<T> numOps, RegressionOptions options)
+    {
+        return new MultivariateRegression<T>(numOps, options);
+    }
+
+    public static MultivariateRegression<T> CreateLassoRegression<T>(INumericOperations<T> numOps, RegressionOptions options)
+    {
+        return new MultivariateRegression<T>(numOps, options);
+    }
+
+    public static MultivariateRegression<T> CreateElasticNetRegression<T>(INumericOperations<T> numOps, ElasticNetRegressionOptions<T> options)
+    {
+        return new MultivariateRegression<T>(numOps, options);
+    }
+}

src/Helpers/StatisticsHelper.cs

@@ -1,4 +1,6 @@
-namespace AiDotNet.Helpers;
+using System.Linq;
+
+namespace AiDotNet.Helpers;
 
 public static class StatisticsHelper<T>
 {
@@ -903,21 +905,21 @@ public static T CalculateTValue(int degreesOfFreedom, T confidenceLevel)
         return CalculateInverseStudentTCDF(degreesOfFreedom, NumOps.Subtract(NumOps.One, alpha));
     }
 
-    public static (T FirstQuartile, T ThirdQuartile) CalculateQuartiles(Vector<T> data)
+    public static (T FirstQuantile, T ThirdQuantile) CalculateQuantiles(Vector<T> data)
     {
         var sortedData = data.OrderBy(x => x).ToArray();
         int n = sortedData.Length;
 
-        T Q1 = CalculateQuartile(sortedData, NumOps.FromDouble(0.25));
-        T Q3 = CalculateQuartile(sortedData, NumOps.FromDouble(0.75));
+        T Q1 = CalculateQuantile(sortedData, NumOps.FromDouble(0.25));
+        T Q3 = CalculateQuantile(sortedData, NumOps.FromDouble(0.75));
 
         return (Q1, Q3);
     }
 
-    public static T CalculateQuartile(T[] sortedData, T quartile)
+    public static T CalculateQuantile(T[] sortedData, T quantile)
     {
         int n = sortedData.Length;
-        T position = NumOps.Multiply(NumOps.FromDouble(n - 1), quartile);
+        T position = NumOps.Multiply(NumOps.FromDouble(n - 1), quantile);
         int index = NumOps.ToInt32(NumOps.Round(position));
         T fraction = NumOps.Subtract(position, NumOps.FromDouble(index));
 
@@ -956,4 +958,117 @@ public static (T skewness, T kurtosis) CalculateSkewnessAndKurtosis(Vector<T> sa
 
         return (skewness, kurtosis);
     }
+
+    public static (T Lower, T Upper) CalculateToleranceInterval(Vector<T> actual, Vector<T> predicted, T confidenceLevel)
+    {
+        int n = actual.Length;
+        T mean = predicted.Average();
+        T stdDev = CalculateStandardDeviation(predicted);
+        T factor = NumOps.FromDouble(Math.Sqrt(1 + (1.0 / n)));
+        T tValue = CalculateTValue(n - 1, confidenceLevel);
+        T margin = NumOps.Multiply(tValue, NumOps.Multiply(stdDev, factor));
+
+        return (NumOps.Subtract(mean, margin), NumOps.Add(mean, margin));
+    }
+
+    public static (T Lower, T Upper) CalculateForecastInterval(Vector<T> actual, Vector<T> predicted, T confidenceLevel)
+    {
+        int n = actual.Length;
+        T mean = predicted.Average();
+        T mse = CalculateMeanSquaredError(actual, predicted);
+        T factor = NumOps.FromDouble(Math.Sqrt(1 + (1.0 / n)));
+        T tValue = CalculateTValue(n - 1, confidenceLevel);
+        T margin = NumOps.Multiply(tValue, NumOps.Multiply(NumOps.Sqrt(mse), factor));
+
+        return (NumOps.Subtract(mean, margin), NumOps.Add(mean, margin));
+    }
+
+    public static List<(T Quantile, T Lower, T Upper)> CalculateQuantileIntervals(Vector<T> actual, Vector<T> predicted, T[] quantiles)
+    {
+        var result = new List<(T Quantile, T Lower, T Upper)>();
+        var sortedPredictions = new Vector<T>([.. predicted.OrderBy(x => x)]);
+
+        foreach (var q in quantiles)
+        {
+            T lowerQuantile = CalculateQuantile(sortedPredictions, NumOps.Subtract(q, NumOps.FromDouble(0.025)));
+            T upperQuantile = CalculateQuantile(sortedPredictions, NumOps.Add(q, NumOps.FromDouble(0.025)));
+            result.Add((q, lowerQuantile, upperQuantile));
+        }
+
+        return result;
+    }
+
+    public static (T Lower, T Upper) CalculateBootstrapInterval(Vector<T> actual, Vector<T> predicted, T confidenceLevel)
+    {
+        int n = actual.Length;
+        int bootstrapSamples = 1000;
+        var bootstrapMeans = new List<T>();
+
+        Random random = new();
+        for (int i = 0; i < bootstrapSamples; i++)
+        {
+            var sample = new Vector<T>(n);
+            for (int j = 0; j < n; j++)
+            {
+                int index = random.Next(n);
+                sample[j] = predicted[index];
+            }
+            bootstrapMeans.Add(sample.Average());
+        }
+
+        bootstrapMeans.Sort();
+        int lowerIndex = NumOps.ToInt32(NumOps.Divide(NumOps.Multiply(confidenceLevel, NumOps.FromDouble(bootstrapSamples)), NumOps.FromDouble(2)));
+        int upperIndex = bootstrapSamples - lowerIndex - 1;
+
+        return (bootstrapMeans[lowerIndex], bootstrapMeans[upperIndex]);
+    }
+
+    public static (T Lower, T Upper) CalculateSimultaneousPredictionInterval(Vector<T> actual, Vector<T> predicted, T confidenceLevel)
+    {
+        int n = actual.Length;
+        T mean = predicted.Average();
+        T mse = CalculateMeanSquaredError(actual, predicted);
+        T factor = NumOps.Sqrt(NumOps.Multiply(NumOps.FromDouble(2), confidenceLevel));
+        T margin = NumOps.Multiply(factor, NumOps.Sqrt(mse));
+
+        return (NumOps.Subtract(mean, margin), NumOps.Add(mean, margin));
+    }
+
+    public static (T Lower, T Upper) CalculateJackknifeInterval(Vector<T> actual, Vector<T> predicted)
+    {
+        int n = actual.Length;
+        var jackknifeSamples = new List<T>();
+
+        for (int i = 0; i < n; i++)
+        {
+            var sample = new Vector<T>(n - 1);
+            int index = 0;
+            for (int j = 0; j < n; j++)
+            {
+                if (j != i)
+                {
+                    sample[index++] = predicted[j];
+                }
+            }
+            jackknifeSamples.Add(sample.Average());
+        }
+
+        T jackknifeEstimate = new Vector<T>([.. jackknifeSamples]).Average();
+        T jackknifeStdError = CalculateStandardDeviation(new Vector<T>([.. jackknifeSamples]));
+        T tValue = CalculateTValue(n - 1, NumOps.FromDouble(0.95));
+        T margin = NumOps.Multiply(tValue, jackknifeStdError);
+
+        return (NumOps.Subtract(jackknifeEstimate, margin), NumOps.Add(jackknifeEstimate, margin));
+    }
+
+    public static (T Lower, T Upper) CalculatePercentileInterval(Vector<T> predicted, T confidenceLevel)
+    {
+        var sortedPredictions = new Vector<T>([.. predicted.OrderBy(x => x)]);
+        int n = sortedPredictions.Length;
+        T alpha = NumOps.Subtract(NumOps.One, confidenceLevel);
+        int lowerIndex = NumOps.ToInt32(NumOps.Divide(NumOps.Multiply(alpha, NumOps.FromDouble(n)), NumOps.FromDouble(2.0)));
+        int upperIndex = n - lowerIndex - 1;
+
+        return (sortedPredictions[lowerIndex], sortedPredictions[upperIndex]);
+    }
 }

src/Interfaces/IOutlierRemoval.cs

@@ -1,12 +1,6 @@
 namespace AiDotNet.Interfaces;
 
-public abstract class IOutlierRemoval
+public interface IOutlierRemoval<T>
 {
-    internal abstract (double[], double[]) RemoveOutliers(double[] rawInputs, double[] rawOutputs);
-
-    internal abstract (double[][], double[]) RemoveOutliers(double[][] rawInputs, double[] rawOutputs);
-
-    internal abstract (double[][], double[][]) RemoveOutliers(double[][] rawInputs, double[][] rawOutputs);
-
-    internal IQuartile? Quartile { get; set; }
+    (Matrix<T> CleanedInputs, Vector<T> CleanedOutputs) RemoveOutliers(Matrix<T> inputs, Vector<T> outputs);
 }

src/Interfaces/IPredictionModelBuilder.cs

@@ -10,6 +10,7 @@ public interface IPredictionModelBuilder<T>
     IPredictionModelBuilder<T> WithRegression(IRegression<T> regression);
     IPredictionModelBuilder<T> WithOptimizer(IOptimizationAlgorithm<T> optimizationAlgorithm, OptimizationAlgorithmOptions optimizationOptions);
     IPredictionModelBuilder<T> WithDataPreprocessor(IDataPreprocessor<T> dataPreprocessor);
+    IPredictionModelBuilder<T> WithOutlierRemoval(IOutlierRemoval<T> outlierRemoval);
     PredictionModelResult<T> Build(Matrix<T> x, Vector<T> y);
     Vector<T> Predict(Matrix<T> newData, PredictionModelResult<T> model);
     void SaveModel(PredictionModelResult<T> model, string filePath);

src/LinearAlgebra/DataPreprocessor.cs

@@ -4,13 +4,15 @@ public class DataPreprocessor<T> : IDataPreprocessor<T>
 {
     private readonly INormalizer<T> _normalizer;
     private readonly IFeatureSelector<T> _featureSelector;
+    private readonly IOutlierRemoval<T> _outlierRemoval;
     private readonly bool _normalizeBeforeFeatureSelection;
     private readonly PredictionModelOptions _options;
 
-    public DataPreprocessor(INormalizer<T> normalizer, IFeatureSelector<T> featureSelector, PredictionModelOptions options)
+    public DataPreprocessor(INormalizer<T> normalizer, IFeatureSelector<T> featureSelector, IOutlierRemoval<T> outlierRemoval, PredictionModelOptions options)
     {
         _normalizer = normalizer;
         _featureSelector = featureSelector;
+        _outlierRemoval = outlierRemoval;
         _options = options;
         _normalizeBeforeFeatureSelection = options.NormalizeBeforeFeatureSelection;
     }
@@ -19,6 +21,8 @@ public DataPreprocessor(INormalizer<T> normalizer, IFeatureSelector<T> featureSe
     {
         NormalizationInfo<T> normInfo = new();
 
+        (X, y) = _outlierRemoval.RemoveOutliers(X, y);
+
         if (_normalizeBeforeFeatureSelection)
         {
             (X, normInfo.XParams) = _normalizer.NormalizeMatrix(X);
@@ -35,8 +39,7 @@ public DataPreprocessor(INormalizer<T> normalizer, IFeatureSelector<T> featureSe
         return (X, y, normInfo);
     }
 
-    public (Matrix<T> XTrain, Vector<T> yTrain, Matrix<T> XValidation, Vector<T> yValidation, Matrix<T> XTest, Vector<T> yTest) 
-        SplitData(Matrix<T> X, Vector<T> y)
+    public (Matrix<T> XTrain, Vector<T> yTrain, Matrix<T> XValidation, Vector<T> yValidation, Matrix<T> XTest, Vector<T> yTest) SplitData(Matrix<T> X, Vector<T> y)
     {
         int totalSamples = X.Rows;
         int trainSize = (int)(totalSamples * _options.TrainingSplitPercentage);

src/LinearAlgebra/Matrix.cs

@@ -131,6 +131,17 @@ public static Matrix<T> CreateZeros(int rows, int cols, INumericOperations<T>? n
         return matrix.Zeros(rows, cols);
     }
 
+    public static Matrix<T> CreateDiagonal(Vector<T> diagonal, INumericOperations<T>? numericOperations = null)
+    {
+        var matrix = new Matrix<T>(diagonal.Length, diagonal.Length, numericOperations);
+        for (int i = 0; i < diagonal.Length; i++)
+        {
+            matrix[i, i] = diagonal[i];
+        }
+
+        return matrix;
+    }
+
     public new static Matrix<T> Empty()
     {
         return new Matrix<T>(0, 0);

src/LinearAlgebra/MatrixBase.cs

@@ -109,16 +109,16 @@ public static MatrixBase<T> Empty()
         return new Matrix<T>(0, 0);
     }
 
-    public virtual VectorBase<T> GetRow(int row)
+    public virtual Vector<T> GetRow(int row)
     {
         ValidateIndices(row, 0);
-        return new Vector<T>(Enumerable.Range(0, cols).Select(col => this[row, col]).ToArray(), ops);
+        return new Vector<T>([.. Enumerable.Range(0, cols).Select(col => this[row, col])], ops);
     }
 
-    public virtual VectorBase<T> GetColumn(int col)
+    public virtual Vector<T> GetColumn(int col)
     {
         ValidateIndices(0, col);
-        return new Vector<T>(Enumerable.Range(0, rows).Select(row => this[row, col]).ToArray(), ops);
+        return new Vector<T>([.. Enumerable.Range(0, rows).Select(row => this[row, col])], ops);
     }
 
     public Matrix<T> SubMatrix(int startRow, int startCol, int numRows, int numCols)

src/LinearAlgebra/PredictionModelBuilder.cs

@@ -3,6 +3,7 @@
 global using AiDotNet.Regularization;
 global using AiDotNet.Optimizers;
 global using AiDotNet.Normalizers;
+global using AiDotNet.OutlierRemoval;
 
 namespace AiDotNet.LinearAlgebra;
 
@@ -18,6 +19,7 @@ public class PredictionModelBuilder<T> : IPredictionModelBuilder<T>
     private IRegression<T>? _regression;
     private IOptimizationAlgorithm<T>? _optimizer;
     private IDataPreprocessor<T>? _dataPreprocessor;
+    private IOutlierRemoval<T>? _outlierRemoval;
 
     public PredictionModelBuilder(PredictionModelOptions? options = null)
     {
@@ -73,6 +75,12 @@ public IPredictionModelBuilder<T> WithDataPreprocessor(IDataPreprocessor<T> data
         return this;
     }
 
+    public IPredictionModelBuilder<T> WithOutlierRemoval(IOutlierRemoval<T> outlierRemoval)
+    {
+        _outlierRemoval = outlierRemoval;
+        return this;
+    }
+
     public PredictionModelResult<T> Build(Matrix<T> x, Vector<T> y)
     {
         // Validate inputs
@@ -93,7 +101,8 @@ public PredictionModelResult<T> Build(Matrix<T> x, Vector<T> y)
         var fitDetector = _fitDetector ?? new DefaultFitDetector<T>();
         var fitnessCalculator = _fitnessCalculator ?? new RSquaredFitnessCalculator<T>();
         var regularization = _regularization ?? new NoRegularization<T>();
-        var dataPreprocessor = _dataPreprocessor ?? new DataPreprocessor<T>(normalizer, featureSelector, _options);
+        var outlierRemoval = _outlierRemoval ?? new NoOutlierRemoval<T>();
+        var dataPreprocessor = _dataPreprocessor ?? new DataPreprocessor<T>(normalizer, featureSelector, outlierRemoval, _options);
 
         // Preprocess the data
         var (preprocessedX, preprocessedY, normInfo) = dataPreprocessor.PreprocessData(x, y);

src/Models/BasicStats.cs

@@ -58,7 +58,7 @@ private void CalculateStats(Vector<T> values)
         Min = values.Min();
         Max = values.Max();
         Median = StatisticsHelper<T>.CalculateMedian(values);
-        (FirstQuartile, ThirdQuartile) = StatisticsHelper<T>.CalculateQuartiles(values);
+        (FirstQuartile, ThirdQuartile) = StatisticsHelper<T>.CalculateQuantiles(values);
         InterquartileRange = NumOps.Subtract(ThirdQuartile, FirstQuartile);
         MAD = StatisticsHelper<T>.CalculateMeanAbsoluteDeviation(values, Median);
     }