Added many new layers and many new neural networks as well as redesigning some major parts of the neural network architecture

Author: ooples

src/ActivationFunctions/LeakyReLUActivation.cs

@@ -0,0 +1,51 @@
+namespace AiDotNet.ActivationFunctions;
+
+public class LeakyReLUActivation<T> : ActivationFunctionBase<T>
+{
+    private readonly T _alpha;
+
+    public LeakyReLUActivation(double alpha = 0.01)
+    {
+        _alpha = NumOps.FromDouble(alpha);
+    }
+
+    protected override bool SupportsScalarOperations() => true;
+
+    public override T Activate(T input)
+    {
+        return NumOps.GreaterThan(input, NumOps.Zero) ? input : NumOps.Multiply(_alpha, input);
+    }
+
+    public override Vector<T> Activate(Vector<T> input)
+    {
+        return input.Transform(x => Activate(x));
+    }
+
+    public override T Derivative(T input)
+    {
+        return NumOps.GreaterThan(input, NumOps.Zero) ? NumOps.One : _alpha;
+    }
+
+    public override Matrix<T> Derivative(Vector<T> input)
+    {
+        int size = input.Length;
+        Matrix<T> jacobian = new Matrix<T>(size, size);
+
+        for (int i = 0; i < size; i++)
+        {
+            for (int j = 0; j < size; j++)
+            {
+                if (i == j)
+                {
+                    jacobian[i, j] = Derivative(input[i]);
+                }
+                else
+                {
+                    jacobian[i, j] = NumOps.Zero;
+                }
+            }
+        }
+
+        return jacobian;
+    }
+}

src/ActivationFunctions/SignActivation.cs

@@ -0,0 +1,95 @@
+namespace AiDotNet.ActivationFunctions;
+
+public class SignActivation<T> : ActivationFunctionBase<T>
+{
+    protected override bool SupportsScalarOperations() => true;
+
+    public override T Activate(T input)
+    {
+        if (NumOps.LessThan(input, NumOps.Zero))
+            return NumOps.FromDouble(-1);
+        else if (NumOps.GreaterThan(input, NumOps.Zero))
+            return NumOps.One;
+        else
+            return NumOps.Zero;
+    }
+
+    public override T Derivative(T input)
+    {
+        // The derivative of the sign function is 0 everywhere except at 0,
+        // where it's undefined. We'll return 0 for all inputs.
+        return NumOps.Zero;
+    }
+
+    public override Vector<T> Activate(Vector<T> input)
+    {
+        Vector<T> output = new Vector<T>(input.Length);
+        for (int i = 0; i < input.Length; i++)
+        {
+            output[i] = Activate(input[i]);
+        }
+
+        return output;
+    }
+
+    public override Matrix<T> Derivative(Vector<T> input)
+    {
+        int n = input.Length;
+        Matrix<T> jacobian = new Matrix<T>(n, n);
+        // The Jacobian matrix will be all zeros
+        for (int i = 0; i < n; i++)
+        {
+            for (int j = 0; j < n; j++)
+            {
+                jacobian[i, j] = NumOps.Zero;
+            }
+        }
+
+        return jacobian;
+    }
+
+    public override Tensor<T> Activate(Tensor<T> input)
+    {
+        Tensor<T> output = new Tensor<T>(input.Shape);
+        int totalElements = input.Shape.Aggregate(1, (a, b) => a * b);
+
+        for (int i = 0; i < totalElements; i++)
+        {
+            output[i] = Activate(input[i]);
+        }
+
+        return output;
+    }
+
+    public override Tensor<T> Derivative(Tensor<T> input)
+    {
+        int[] outputShape = new int[input.Shape.Length + 1];
+        Array.Copy(input.Shape, outputShape, input.Shape.Length);
+        outputShape[outputShape.Length - 1] = input.Shape[input.Shape.Length - 1];
+
+        Tensor<T> output = new Tensor<T>(outputShape);
+        int batchSize = input.Shape[0];
+        int vectorLength = input.Shape[1];
+
+        for (int i = 0; i < batchSize; i++)
+        {
+            Vector<T> vector = new Vector<T>(vectorLength);
+            for (int j = 0; j < vectorLength; j++)
+            {
+                vector[j] = input[i, j];
+            }
+
+            Matrix<T> jacobian = Derivative(vector);
+
+            for (int j = 0; j < vectorLength; j++)
+            {
+                for (int k = 0; k < vectorLength; k++)
+                {
+                    output[i, j, k] = jacobian[j, k];
+                }
+            }
+        }
+
+        return output;
+    }
+}

src/Enums/ActivationType.cs

@@ -0,0 +1,10 @@
+namespace AiDotNet.Enums;
+
+public enum ActivationType
+{
+    Softmax,
+    Sigmoid,
+    ReLU,
+    Tanh,
+    Other
+}

src/Enums/InputType.cs

@@ -0,0 +1,8 @@
+namespace AiDotNet.Enums;
+
+public enum InputType
+{
+    OneDimensional,
+    TwoDimensional,
+    ThreeDimensional
+}

src/Enums/NetworkComplexity.cs

@@ -0,0 +1,32 @@
+namespace AiDotNet.Enums;
+
+/// <summary>
+/// Defines the complexity level of a neural network architecture.
+/// </summary>
+public enum NetworkComplexity
+{
+    /// <summary>
+    /// Simple network with minimal layers, suitable for basic tasks.
+    /// </summary>
+    Simple,
+
+    /// <summary>
+    /// Medium complexity network with a moderate number of layers.
+    /// </summary>
+    Medium,
+
+    /// <summary>
+    /// Deep network with many layers, suitable for complex tasks.
+    /// </summary>
+    Deep,
+
+    /// <summary>
+    /// Very deep network with extensive layers and connections.
+    /// </summary>
+    VeryDeep,
+
+    /// <summary>
+    /// Custom complexity defined by the user.
+    /// </summary>
+    Custom
+}

src/Enums/NeuralNetworkTaskType.cs

@@ -0,0 +1,117 @@
+namespace AiDotNet.Enums;
+
+/// <summary>
+/// Defines the different types of tasks that a neural network can be designed to perform.
+/// </summary>
+public enum NeuralNetworkTaskType
+{
+    /// <summary>
+    /// Binary classification task (two classes)
+    /// </summary>
+    BinaryClassification,
+    
+    /// <summary>
+    /// Multi-class classification task (more than two classes)
+    /// </summary>
+    MultiClassClassification,
+    
+    /// <summary>
+    /// Multi-label classification task (multiple labels can be assigned)
+    /// </summary>
+    MultiLabelClassification,
+    
+    /// <summary>
+    /// Regression task (predicting continuous values)
+    /// </summary>
+    Regression,
+    
+    /// <summary>
+    /// Sequence-to-sequence task (e.g., machine translation)
+    /// </summary>
+    SequenceToSequence,
+
+    /// <summary>
+    /// Sequence-to-sequence classification
+    /// </summary>
+    SequenceClassification,
+    
+    /// <summary>
+    /// Time series forecasting task
+    /// </summary>
+    TimeSeriesForecasting,
+    
+    /// <summary>
+    /// Image classification task
+    /// </summary>
+    ImageClassification,
+    
+    /// <summary>
+    /// Object detection task
+    /// </summary>
+    ObjectDetection,
+    
+    /// <summary>
+    /// Image segmentation task
+    /// </summary>
+    ImageSegmentation,
+    
+    /// <summary>
+    /// Natural language processing task
+    /// </summary>
+    NaturalLanguageProcessing,
+    
+    /// <summary>
+    /// Text generation task
+    /// </summary>
+    TextGeneration,
+    
+    /// <summary>
+    /// Reinforcement learning task
+    /// </summary>
+    ReinforcementLearning,
+    
+    /// <summary>
+    /// Anomaly detection task
+    /// </summary>
+    AnomalyDetection,
+    
+    /// <summary>
+    /// Recommendation system task
+    /// </summary>
+    Recommendation,
+    
+    /// <summary>
+    /// Clustering task
+    /// </summary>
+    Clustering,
+    
+    /// <summary>
+    /// Dimensionality reduction task
+    /// </summary>
+    DimensionalityReduction,
+    
+    /// <summary>
+    /// Generative task (e.g., GANs, VAEs)
+    /// </summary>
+    Generative,
+    
+    /// <summary>
+    /// Speech recognition task
+    /// </summary>
+    SpeechRecognition,
+    
+    /// <summary>
+    /// Audio processing task
+    /// </summary>
+    AudioProcessing,
+
+    /// <summary>
+    /// Language translation
+    /// </summary>
+    Translation,
+    
+    /// <summary>
+    /// Custom task type
+    /// </summary>
+    Custom
+}

src/Enums/SamplingType.cs

@@ -0,0 +1,8 @@
+namespace AiDotNet.Enums;
+
+public enum SamplingType
+{
+    Max,
+    Average,
+    L2Norm
+}

src/Enums/SpikingNeuronType.cs

@@ -0,0 +1,10 @@
+namespace AiDotNet.Enums;
+
+public enum SpikingNeuronType
+{
+    LeakyIntegrateAndFire,
+    IntegrateAndFire,
+    Izhikevich,
+    HodgkinHuxley,
+    AdaptiveExponential
+}

src/Enums/TransformerTaskType.cs

@@ -0,0 +1,10 @@
+namespace AiDotNet.Enums;
+
+public enum TransformerTaskType
+{
+    Classification,
+    Regression,
+    TextGeneration,
+    SequenceTagging,
+    Translation
+}

src/Extensions/SerializationExtensions.cs

@@ -56,4 +56,25 @@ public static object ReadValue(this BinaryReader reader, Type type)
             return reader.ReadBoolean();
         throw new ArgumentException($"Unsupported type: {type}");
     }
+
+    public static int[] ReadInt32Array(this BinaryReader reader)
+    {
+        int length = reader.ReadInt32();
+        int[] array = new int[length];
+        for (int i = 0; i < length; i++)
+        {
+            array[i] = reader.ReadInt32();
+        }
+
+        return array;
+    }
+
+    public static void WriteInt32Array(this BinaryWriter writer, int[] array)
+    {
+        writer.Write(array.Length);
+        foreach (int value in array)
+        {
+            writer.Write(value);
+        }
+    }
 }