Multiple convolution neural networks approach for online handwriting recognition

using System;
using System.Collections.Generic;
using System.Text;
using System.Xml.Serialization;  // Does XML serializing for a class.
using System.IO;			     // Required for using Memory stream objects.
using System.Drawing;
namespace ANN.Perceptron.Common
{
    public enum LossFunctions
    {
        CrossEntropy = 0,
        LogisticRegression = 1,
        MeanSquareError = 2,
        NegativeLogLikelihood = 3,
    }
    public enum LayerTypes
    {
        Input = 0,
        ConvolutionalSubsampling = 1,      // Patrice Simards layertype
        Convolution = 4,
        Sampling = 5,
        FullyConnected = 2,
        Output=3,
        
    }
    public enum ActivationFunctions
    {
        AbsTanh = 0,
        AveragePoolingTanh = 1,
        Gaussian = 2,
        Linear = 3,
        Logistics = 4,
        MaxPoolingTanh = 5,
        MedianPoolingTanh = 6,
        None = 7,
        Tanh = 8,
    }
    /// References: 
    /// XML Serialization at http://samples.gotdotnet.com/:
    /// http://samples.gotdotnet.com/QuickStart/howto/default.aspx?url=/quickstart/howto/doc/xmlserialization/rwobjfromxml.aspx
    /// 
    /// How do I serialize an image file as XML in .NET?
    /// http://www.perfectxml.com/Answers.asp?ID=2
    /// </summary>
    [XmlRootAttribute("NetworkParameters", Namespace = "NeuralNetwork", IsNullable = false)]
    public class NetworkParameters
    {
        public Size DesignedPatternSize;
        public Size RealPatternSize;
        // for limiting the step size in backpropagation, since we are using second order
        // "Stochastic Diagonal Levenberg-Marquardt" update algorithm.  See Yann LeCun 1998
        // "Gradianet-Based Learning Applied to Document Recognition" at page 41
        public double MicronLimitParameter { get; set; }
        public uint NumHessianPatterns { get; set; }
        // for distortions of the input image, in an attempt to improve generalization
        public double MaxScaling { get; set; }  // as a percentage, such as 20.0 for plus/minus 20%
        public double MaxRotation { get; set; }  // in degrees, such as 20.0 for plus/minus rotations of 20 degrees
        public double ElasticSigma { get; set; }  // one sigma value for randomness in Simard's elastic distortions
        public double ElasticScaling { get; set; }  // after-smoohting scale factor for Simard's elastic distortions
        public int Epochs { get; set; }
        public int DecayAfterEpochs { get; set; }
        public int DistotionEpochs { get; set; }
        public double WeightSaveTreshold { get; set; }
        public bool Distorted { get; set; }
        public int SameDistortionsForNEpochs { get; set; }
        public double SeverityFactor { get; set; }
        public double InitialEtaLearningRate { get; set; }
        public double LearningRateDecay { get; set; }
        public double MinimumEtaLearningRate { get; set; }
        public uint AfterEveryNBackprops { get; set; }
        public double EtaDecay { get; set; }
        
        ////////////
        public NetworkParameters()
        {
            InitialEtaLearningRate = 0.001;
            LearningRateDecay = 0.794328235;  // 0.794328235 = 0.001 down to 0.00001 in 20 epochs 
            MinimumEtaLearningRate = 0.00001;
            AfterEveryNBackprops = 0;
            // parameters for controlling distortions of input image
            MaxScaling = 15.0;  // like 20.0 for 20%
            MaxRotation = 15.0;  // like 20.0 for 20 degrees
            ElasticSigma = 8.0;  // higher numbers are more smooth and less distorted; Simard uses 4.0
            ElasticScaling = 0.5;  // higher numbers amplify the distortions; Simard uses 34 (sic, maybe 0.34 ??)
            // for limiting the step size in backpropagation, since we are using second order
            // "Stochastic Diagonal Levenberg-Marquardt" update algorithm.  See Yann LeCun 1998
            // "Gradient-Based Learning Applied to Document Recognition" at page 41
            //MicronLimitParameter = 0.10;  // since we divide by this, update can never be more than 10x current eta
            NumHessianPatterns = 1000;  // number of patterns used to calculate the diagonal Hessian
            DesignedPatternSize = Size.Empty;
            RealPatternSize = Size.Empty;
            SeverityFactor = 0.65;
        }
    }
}