Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Multiple convolution neural networks approach for online handwriting recognition

, 9 Apr 2013 CPOL
The research focuses on the presentation of word recognition technique for an online handwriting recognition system which uses multiple component neural networks (MCNN) as the exchangeable parts of the classifier.
capital_letter_v2.zip
capital letter v2.nnt
digit_v2.zip
digit v2.nnt
lower_case_letter_v2.zip
lower case letter v2.nnt
UNIPENviewer_source_code-noexe.zip
UNIPENviewer
DocToolkit
DocToolkit.csproj.user
DocToolkit.snk
obj
Debug
Release
DrawTools
about.bmp
App.ico
bin
Debug
Release
DrawTools.csproj.user
ellipse.bmp
Ellipse.cur
line.bmp
Line.cur
new.bmp
obj
Debug
DrawTools.DrawArea.resources
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
Release
DrawTools.DrawArea.resources
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
open.bmp
pencil.bmp
Pencil.cur
pointer.bmp
PolyHandle.cur
rectangle.bmp
Rectangle.cur
save.bmp
NNControl
bin
Debug
Common
NNTesting
NNTraining
obj
Debug
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NNControl.Common.UPTemplateControl.resources
NNControl.FlashForm.resources
NNControl.NNTesting.NNTestingControl.resources
NNControl.NNTesting.TextSpellControl.resources
NNControl.NNTraining.ConvolutionForm.resources
NNControl.NNTraining.CreateNetworkForm.resources
NNControl.NNTraining.FullConnectedForm.resources
NNControl.NNTraining.InputLayerForm.resources
NNControl.NNTraining.OutputLayerForm.resources
NNControl.NNTraining.UP_NNTrainingControl.resources
NNControl.Properties.Resources.resources
NNControl.TrainingParametersForm.resources
NNControl.UPViewer.UpImageViewer.resources
UPControl.Common.BaseControl.resources
UPControl.Common.UPTemplateControl.resources
UPControl.FlashForm.resources
UPControl.NNTraining.UP_NNTrainingControl.resources
UPControl.TrainingParametersForm.resources
UPControl.UPViewer.UpImageViewer.resources
UP_NeuralTraining.FlashForm.resources
UP_NeuralTraining.TrainingParametersForm.resources
UP_NeuralTraining.UP_NNTrainingControl.resources
Release
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NNControl.Common.UPTemplateControl.resources
NNControl.FlashForm.resources
NNControl.NNTesting.NNTestingControl.resources
NNControl.NNTraining.ConvolutionForm.resources
NNControl.NNTraining.CreateNetworkForm.resources
NNControl.NNTraining.FullConnectedForm.resources
NNControl.NNTraining.InputLayerForm.resources
NNControl.NNTraining.OutputLayerForm.resources
NNControl.NNTraining.UP_NNTrainingControl.resources
NNControl.Properties.Resources.resources
NNControl.TrainingParametersForm.resources
NNControl.UPViewer.UpImageViewer.resources
Properties
Resources
btnBack.png
btnDrag.png
btnFitToScreen.png
btnNext.png
btnOpen.png
btnPreview.png
btnRotate270.png
btnRotate90.png
btnSelect.png
btnZoomIn.png
btnZoomOut.png
circle.png
clear.png
color_line.png
cry.png
document-new.png
Drag.cur
draw_line.png
ellipse.png
export.png
file.png
fingerprint-recognition.png
folder-open.png
folder.png
folders_explorer.png
Grab.cur
home.png
label-link.png
pointer.png
rectangle.png
save_accept.png
script_(stop).gif
smile.png
stock_draw-line.png
Stop sign.png
Upload.png
user-group-new.png
UPViewer
Perceptron
ActivationFunction
ArchiveSerialization
bin
Debug
Common
Connections
Layers
Network
Neurons
obj
Debug
ANN.Perceptron.Common.BaseControl.resources
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
Neurons.BaseControl.resources
Release
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
Neurons.BaseControl.resources
Properties
Weights
SpellChecker
bin
Debug
Controls
Dictionary
Affix
Dictionary.bmp
Phonetic
doc
NetSpell.ndoc
Forms
Interactive.bmp
obj
Debug
ResGen.read.1.tlog
ResGen.write.1.tlog
SpellChecker.Dictionary.WordDictionary.resources
SpellChecker.Forms.OptionForm.resources
SpellChecker.MultipleSpelling.resources
SpellChecker.NewSpelling.resources
Spell.snk
Spelling.bmp
UNIPENviewer.suo
UNIPENviewer.v11.suo
UNIPENviewer
bin
Debug
Config
de-DE.dic
en-US.dic
fr-FR.dic
it-IT.dic
UNIPENviewer.vshost.exe.manifest
Release
Config
obj
Debug
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
UNIPENviewer.MainForm.resources
UNIPENviewer.Properties.Resources.resources
Release
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
UNIPENviewer.MainForm.resources
UNIPENviewer.Properties.Resources.resources
x86
Debug
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
UNIPENviewer.Form1.resources
UNIPENviewer.Properties.Resources.resources
Properties
Settings.settings
UPImage
Common
Data
FileFormat
obj
Debug
Release
Properties
UNIPENviewer_source_code.zip
bin
Debug
DocToolkit.dll
DocToolkit.pdb
Release
DocToolkit.dll
DocToolkit.csproj.user
DocToolkit.snk
DesignTimeResolveAssemblyReferencesInput.cache
DocToolkit.dll
DocToolkit.pdb
TempPE
DesignTimeResolveAssemblyReferencesInput.cache
DocToolkit.dll
about.bmp
App.ico
DocToolkit.dll
DocToolkit.pdb
DrawTools.dll
DrawTools.pdb
DocToolkit.dll
DrawTools.dll
DrawTools.csproj.user
ellipse.bmp
Ellipse.cur
line.bmp
Line.cur
new.bmp
DesignTimeResolveAssemblyReferencesInput.cache
DrawTools.csproj.GenerateResource.Cache
DrawTools.csprojResolveAssemblyReference.cache
DrawTools.dll
DrawTools.DrawArea.resources
DrawTools.pdb
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
ResolveAssemblyReference.cache
TempPE
DesignTimeResolveAssemblyReferencesInput.cache
DrawTools.dll
DrawTools.DrawArea.resources
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
ResolveAssemblyReference.cache
open.bmp
pencil.bmp
Pencil.cur
pointer.bmp
PolyHandle.cur
rectangle.bmp
Rectangle.cur
Resources
save.bmp
DocToolkit.dll
DocToolkit.pdb
DrawTools-LinhLam.dll
DrawTools-LinhLam.pdb
DrawTools.dll
DrawTools.pdb
NNControl-LinhLam.dll
NNControl-LinhLam.pdb
NNControl.dll
NNControl.pdb
Perceptron.dll
Perceptron.pdb
SpellChecker.dll
SpellChecker.pdb
UP-NeuralTraining.dll
UP-NeuralTraining.pdb
UPImage-LinhLam.dll
UpImage.dll
UpImage.pdb
Release
DocToolkit.dll
DrawTools.dll
Neurons.dll
Neurons.pdb
NNControl.dll
NNControl.pdb
UPImage.dll
UPImage.pdb
DesignTimeResolveAssemblyReferences.cache
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NNControl-LinhLam.dll
NNControl-LinhLam.pdb
NNControl.Common.UPTemplateControl.resources
NNControl.csproj.GenerateResource.Cache
NNControl.csprojResolveAssemblyReference.cache
NNControl.dll
NNControl.FlashForm.resources
NNControl.NNTesting.NNTestingControl.resources
NNControl.NNTesting.TextSpellControl.resources
NNControl.NNTraining.ConvolutionForm.resources
NNControl.NNTraining.CreateNetworkForm.resources
NNControl.NNTraining.FullConnectedForm.resources
NNControl.NNTraining.InputLayerForm.resources
NNControl.NNTraining.OutputLayerForm.resources
NNControl.NNTraining.UP_NNTrainingControl.resources
NNControl.pdb
NNControl.Properties.Resources.resources
NNControl.TrainingParametersForm.resources
NNControl.UPViewer.UpImageViewer.resources
ResolveAssemblyReference.cache
TempPE
Properties.Resources.Designer.cs.dll
UP-NeuralTraining.dll
UP-NeuralTraining.pdb
UPControl.Common.BaseControl.resources
UPControl.Common.UPTemplateControl.resources
UPControl.FlashForm.resources
UPControl.NNTraining.UP_NNTrainingControl.resources
UPControl.TrainingParametersForm.resources
UPControl.UPViewer.UpImageViewer.resources
UP_NeuralTraining.FlashForm.resources
UP_NeuralTraining.TrainingParametersForm.resources
UP_NeuralTraining.UP_NNTrainingControl.resources
DesignTimeResolveAssemblyReferences.cache
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NNControl.Common.UPTemplateControl.resources
NNControl.dll
NNControl.FlashForm.resources
NNControl.NNTesting.NNTestingControl.resources
NNControl.NNTraining.ConvolutionForm.resources
NNControl.NNTraining.CreateNetworkForm.resources
NNControl.NNTraining.FullConnectedForm.resources
NNControl.NNTraining.InputLayerForm.resources
NNControl.NNTraining.OutputLayerForm.resources
NNControl.NNTraining.UP_NNTrainingControl.resources
NNControl.pdb
NNControl.Properties.Resources.resources
NNControl.TrainingParametersForm.resources
NNControl.UPViewer.UpImageViewer.resources
ResolveAssemblyReference.cache
TempPE
Properties.Resources.Designer.cs.dll
btnBack.png
btnDrag.png
btnFitToScreen.png
btnNext.png
btnOpen.png
btnPreview.png
btnRotate270.png
btnRotate90.png
btnSelect.png
btnZoomIn.png
btnZoomOut.png
circle.png
clear.png
color_line.png
cry.png
document-new.png
Drag.cur
draw_line.png
ellipse.png
export.png
file.png
fingerprint-recognition.png
folder-open.png
folder.png
folders_explorer.png
Grab.cur
home.png
label-link.png
pointer.png
rectangle.png
save_accept.png
script_(stop).gif
smile.png
stock_draw-line.png
Stop sign.png
Upload.png
user-group-new.png
AForgeLibrary.dll
AForgeLibrary.pdb
NeuralNetworkLibrary.dll
NeuralNetworkLibrary.pdb
Perceptron.dll
Perceptron.pdb
SpellChecker.dll
SpellChecker.pdb
UpImage.dll
UpImage.pdb
Release
NeuralNetworkLibrary.dll
NeuralNetworkLibrary.pdb
Neurons.dll
Neurons.pdb
UPImage.dll
UPImage.pdb
ANN.Perceptron.Common.BaseControl.resources
ANN.Perceptron.csproj.GenerateResource.Cache
ANN.Perceptron.csprojResolveAssemblyReference.cache
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NeuralNetworkLibrary.dll
NeuralNetworkLibrary.pdb
Neurons.BaseControl.resources
Perceptron.dll
Perceptron.pdb
ResolveAssemblyReference.cache
TempPE
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
NeuralNetworkLibrary.dll
NeuralNetworkLibrary.pdb
Neurons.BaseControl.resources
Neurons.dll
Neurons.pdb
ResolveAssemblyReference.cache
Service References
SpellChecker.dll
SpellChecker.pdb
Release
Dictionary.bmp
NetSpell.ndoc
Interactive.bmp
DesignTimeResolveAssemblyReferences.cache
DesignTimeResolveAssemblyReferencesInput.cache
NetSpell.SpellChecker.dll
NetSpell.SpellChecker.pdb
ResGen.read.1.tlog
ResGen.write.1.tlog
SpellChecker.csproj.GenerateResource.Cache
SpellChecker.csprojResolveAssemblyReference.cache
SpellChecker.Dictionary.WordDictionary.resources
SpellChecker.dll
SpellChecker.Forms.OptionForm.resources
SpellChecker.MultipleSpelling.resources
SpellChecker.NewSpelling.resources
SpellChecker.pdb
TempPE
Spell.snk
Spelling.bmp
UNIPENviewer.suo
UNIPENviewer.v11.suo
de-DE.dic
DocToolkit.dll
DocToolkit.pdb
DrawTools-LinhLam.dll
DrawTools-LinhLam.pdb
DrawTools.dll
DrawTools.pdb
en-US.dic
fr-FR.dic
it-IT.dic
NNControl-LinhLam.dll
NNControl.dll
NNControl.pdb
Perceptron.dll
Perceptron.pdb
SpellChecker.dll
SpellChecker.pdb
UNIPENviewer-LinhLam.exe
UNIPENviewer-LinhLam.pdb
UNIPENviewer.exe
UNIPENviewer.pdb
UNIPENviewer.vshost.exe
UNIPENviewer.vshost.exe.manifest
UPImage-LinhLam.dll
UPImage-LinhLam.pdb
UpImage.dll
UpImage.pdb
DesignTimeResolveAssemblyReferences.cache
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
ResolveAssemblyReference.cache
TempPE
Properties.Resources.Designer.cs.dll
UNIPENviewer-LinhLam.exe
UNIPENviewer-LinhLam.pdb
UNIPENviewer.csproj.GenerateResource.Cache
UNIPENviewer.csprojResolveAssemblyReference.cache
UNIPENviewer.exe
UNIPENviewer.MainForm.resources
UNIPENviewer.pdb
UNIPENviewer.Properties.Resources.resources
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
ResolveAssemblyReference.cache
UNIPENviewer.exe
UNIPENviewer.MainForm.resources
UNIPENviewer.pdb
UNIPENviewer.Properties.Resources.resources
DesignTimeResolveAssemblyReferences.cache
DesignTimeResolveAssemblyReferencesInput.cache
GenerateResource.read.1.tlog
GenerateResource.write.1.tlog
ResolveAssemblyReference.cache
UNIPENviewer.exe
UNIPENviewer.Form1.resources
UNIPENviewer.pdb
UNIPENviewer.Properties.Resources.resources
Settings.settings
bin
Debug
AForgeLibrary.dll
AForgeLibrary.pdb
UPDatabase.dll
UPDatabase.pdb
UpImage-LinhLam.dll
UpImage-LinhLam.pdb
UpImage.dll
UpImage.pdb
UPUnipenLib.dll
UPUnipenLib.pdb
Release
UPImage.dll
UPImage.pdb
DesignTimeResolveAssemblyReferencesInput.cache
TempPE
UPDatabase.dll
UPDatabase.pdb
UpImage-LinhLam-2.pdb
UpImage-LinhLam.dll
UPImage-LinhLam.pdb
UpImage.dll
UpImage.pdb
UPUnipenLib.dll
UPUnipenLib.pdb
DesignTimeResolveAssemblyReferencesInput.cache
UPImage.dll
UPImage.pdb
Word_samples.zip
Word samples
beach.dtl
Bengal.dtl
best.dtl
Brower.dtl
Buy.dtl
Byte byte.dtl
Change.dtl
Color.dtl
Company.dtl
Complet.dtl
copy.dtl
cup.dtl
draw tool.dtl
Eastern.dtl
Eat.dtl
eldest.dtl
Emergency.dtl
English.dtl
File.dtl
Finist.dtl
found nothing.dtl
France.dtl
Future.dtl
help me now.dtl
Hey hello.dtl
Hobby.dtl
How are you.dtl
Huck.dtl
icon.dtl
inker.dtl
Internet.dtl
Link.dtl
load.dtl
Local.dtl
Lonely.dtl
loosen.dtl
love you.dtl
Madden.dtl
Main gate.dtl
Mercy.dtl
Module.dtl
monday.dtl
Moon.dtl
mouse.dtl
my turn.dtl
net spell.dtl
network.dtl
never.dtl
newest.dtl
noted.dtl
Novel.dtl
oldest.dtl
Option.dtl
Pencil.dtl
petro.dtl
Pink.dtl
quick and slow.dtl
Rock.dtl
save.dtl
Sell.dtl
slam.dtl
smart phone.dtl
Strong.dtl
Strongest.dtl
success.dtl
Summer.dtl
Take.dtl
Text.dtl
Took.dtl
Train.dtl
Tuesday.dtl
Valence.dtl
Victory.dtl
viewer.dtl
vintage.dtl
Volume.dtl
water.dtl
Weak.dtl
Window.dtl
Windy.dtl
word expert.dtl
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;
using ANN.Perceptron.ArchiveSerialization;
using ANN.Perceptron.Common;
using ANN.Perceptron.Neurons;
using System.Threading;
using System.Threading.Tasks;
using UPImage.Data;
namespace ANN.Perceptron.Network
{
    public class TrainingResult 
    {
        public int Epoch;
        public double CurrentMSE;
        public int MisPattern;
        public double Ratio;
        public String Duration;
        public bool Distored;
        public double EtaLearningRate;
        public TrainingResult()
        {
            CurrentMSE = 0;
            Distored = true;
            Duration ="";
            Epoch = 0;
            MisPattern = 0;
            Ratio = 0;
        }

    }
    public class PatternTraining : BackPropagation
    {
        /// <summary>
        /// 
        /// </summary>
        /// 
   #region Parametters

        //backpropagation and training-related members
        public double EstimatedCurrentMSE;  // this number will be changed by one thread and used by others
        public uint MisrecognitionCount;
        public bool IsNeedHessian;
        private int[] RandomTrainingPattern;
        /// <summary>
        /// 
        /// </summary>
      
        uint _iEpochsCompleted;
        double _dMSE = 0;
        int _nn;
        ANN.Perceptron.Common.BaseControl parentControl;
        /// <summary>
        /// 
        /// </summary>
   
   #endregion

        #region Main function
        public PatternTraining(ConvolutionNetwork neuronNet, ByteImageData[] trainingSet, NetworkParameters parameters, bool trainingDataReady,
                            ANN.Perceptron.Common.BaseControl ct)
            : base(neuronNet)
        {
            CurrentPattern = 0;
            _bDataReady = trainingDataReady;
            network = neuronNet;
            patternsData = trainingSet;
            Parameters = parameters;
            Letters = neuronNet.TagetOutputs;
            PatternCount = patternsData.Count();
            inputImageSize = parameters.RealPatternSize;
            RandomTrainingPattern = new int[PatternCount];
            parentControl = ct;
            MisrecognitionCount = 0;
            IsNeedHessian = true;
            Backprops = 0;
            _dMSE = 0;
            _nn = 0;
            if (neuronNet.InputDesignedPatternSize != parameters.DesignedPatternSize)
            {
                parameters.DesignedPatternSize = neuronNet.InputDesignedPatternSize;
            }
            if (parameters.AfterEveryNBackprops == 0)
            {
                parameters.AfterEveryNBackprops = (uint)PatternCount;
            }
            HPTime = new HiPerfTimer();
            
            EtaLearningRate = parameters.InitialEtaLearningRate;
            Initialize();
        }
        #endregion
        protected void Initialize()
        {
            for (int i = 0; i < PatternCount; i++)
            {
                RandomTrainingPattern[i] = i;
            }
            //get Gaussian kernnel
            GetGaussianKernel(Parameters.ElasticSigma);
        }
    
        public void RandomizePatternSequence()
        {

            int l;
            int k;
            ThreadSafeRandom RandomGenerator = new ThreadSafeRandom();
            for (int i = 0; i < PatternCount; i++)
            {
                l = RandomGenerator.Next(0, PatternCount);
                k = RandomTrainingPattern[i];
                RandomTrainingPattern[i] = RandomTrainingPattern[l];
                RandomTrainingPattern[l] = k;
            }
        }
        /// <summary>
        /// 
        /// </summary>
        /// <param name="inputVector"></param>
        void CalculateHessian(CancellationToken token)
        {
            // controls the Neural network's calculation if the diagonal Hessian for the Neural net
            // This will be called from a thread, so although the calculation is lengthy, it should not interfere
            // with the UI

            // we need the neural net exclusively during this calculation, so grab it now
            int nInput = network.Layers.First().NeuronCount;
            int nOutput = network.Layers.Last().NeuronCount;
            double[] inputVector = new double[nInput];  // note: 29x29, not 28x28

            double[] targetOutputVector = new double[nOutput];
            double[] actualOutputVector = new double[nOutput];
            
            Parallel.For(0, nInput, i =>
            {
                inputVector[i] = 0.0;
            });
            Parallel.For(0, nOutput, j =>
            {
                targetOutputVector[j] = 0.0;
                actualOutputVector[j] = 0.0;
            });

          
  
            // calculate the diagonal Hessian using 500 random patterns, per Yann LeCun 1998 "Gradient-Based Learning
            // Applied To Document Recognition"
            string s = "Commencing Caculation of Hessian...";
            // Make synchronous call to main form.
            // MainForm.AddString function runs in main thread.
            // To make asynchronous call use BeginInvoke
            if (parentControl != null)
                parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 2, s });

            // some of this code is similar to the BackpropagationThread() code

            EraseHessianInformation();

            uint numPatternsSampled = Parameters.NumHessianPatterns;
            RandomizePatternSequence();
            for (int kk = 0; kk < numPatternsSampled; ++kk)
            {
                int iRandomPatternNum;

                iRandomPatternNum = RandomTrainingPattern[kk];
                char label = patternsData[iRandomPatternNum].Label;
                //check if network have unkown output
                if (network.IsUnknownOutput)
                {
                    #region isUnknownOutput

                    byte[] grayLevels = patternsData[iRandomPatternNum].Image;
                    // top row of inputVector is left as zero, left-most column is left as zero 
                    Size smallestSize = new Size();
                    if (Parameters.DesignedPatternSize.Width >= Parameters.RealPatternSize.Width)
                    {
                        smallestSize.Width = Parameters.RealPatternSize.Width;
                    }
                    else
                    {
                        smallestSize.Width = Parameters.DesignedPatternSize.Width;
                    }
                    if (Parameters.DesignedPatternSize.Height >= Parameters.RealPatternSize.Height)
                    {

                        smallestSize.Height = Parameters.RealPatternSize.Height;
                    }
                    else
                    {

                        smallestSize.Height = Parameters.DesignedPatternSize.Height;
                    }
                    /*TODO: Check potentially-changing upper bound expression "NNDefinations.g_cImageSize" which is now called only *once*,
                    to ensure the new Parallel.For call matches behavior in the original for-loop
                   (where this upper bound expression had previously been evaluated at the start of *every* loop iteration).*/
                    Parallel.For(0, nInput, ii =>
                    {
                        inputVector[ii] = (double)255.0 / 128.0 - 1.0;
                        // one is white, -one is black
                    });
                    Parallel.For(0, smallestSize.Height, ii =>
                    {
                        for (int jj = 0; jj < smallestSize.Width; ++jj)
                        {
                            inputVector[jj + Parameters.DesignedPatternSize.Width * ii] = (double)((int)(byte)grayLevels[jj +
                               Parameters.RealPatternSize.Width * ii]) / 128.0 - 1.0; // one is white, -one is black
                        }
                    });

                   // desired output vector

                    Parallel.For(0, nOutput, ii =>
                    {
                        targetOutputVector[ii] = -1.0;
                    });
                    if (Letters.Contains(label))
                    {
                        targetOutputVector[Letters.IndexOf(label)] = 1.0;
                    }
                    else
                    {
                        targetOutputVector[nOutput - 1] = 1.0;
                    }
                    // apply distortion map to inputVector.  It's not certain that this is needed or helpful.
                    // The second derivatives do NOT rely on the output of the neural net (i.e., because the 
                    // second derivative of the MSE function is exactly 1 (one), regardless of the actual output
                    // of the net).  However, since the backpropagated second derivatives rely on the outputs of
                    // each neuron, distortion of the pattern might reveal previously-unseen information about the
                    // nature of the Hessian.  But I am reluctant to give the full distortion, so I set the
                    // severityFactor to only 2/3 approx
                    GenerateDistortionMap(Parameters.SeverityFactor);
                    ApplyDistortionMap(inputVector);
                    // forward calculate the neural network

                    Calculate(inputVector, (int)nInput, actualOutputVector, (int)nOutput, null);
                    // backpropagate the second derivatives
                    BackpropagateSecondDervatives(actualOutputVector, targetOutputVector, nOutput);
                    double ratio = (double)((double)kk / numPatternsSampled);
                    int n = (int)Math.Floor(ratio * 100);
                    if (parentControl != null)
                        parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 3, n });
                    //
                    if (token.IsCancellationRequested)
                    {
                        return;
                    }
                   
                    #endregion
                }
                else
                {
                    if (Letters.Contains(label))
                    {
                        byte[] grayLevels = patternsData[iRandomPatternNum].Image;
                        // top row of inputVector is left as zero, left-most column is left as zero 
                        Size smallestSize = new Size();
                        if (Parameters.DesignedPatternSize.Width >= Parameters.RealPatternSize.Width)
                        {
                            smallestSize.Width = Parameters.RealPatternSize.Width;
                        }
                        else
                        {
                            smallestSize.Width = Parameters.DesignedPatternSize.Width;
                        }
                        if (Parameters.DesignedPatternSize.Height >= Parameters.RealPatternSize.Height)
                        {

                            smallestSize.Height = Parameters.RealPatternSize.Height;
                        }
                        else
                        {

                            smallestSize.Height = Parameters.DesignedPatternSize.Height;
                        }
                        /*TODO: Check potentially-changing upper bound expression "NNDefinations.g_cImageSize" which is now called only *once*,
                        to ensure the new Parallel.For call matches behavior in the original for-loop
                       (where this upper bound expression had previously been evaluated at the start of *every* loop iteration).*/
                        Parallel.For(0, nInput, ii =>
                        {
                            inputVector[ii] = (double)255.0 / 128.0 - 1.0;
                            // one is white, -one is black
                        });
                        Parallel.For(0, smallestSize.Height, ii =>
                        {
                            for (int jj = 0; jj < smallestSize.Width; ++jj)
                            {
                                inputVector[jj + Parameters.DesignedPatternSize.Width * ii] = (double)((int)(byte)grayLevels[jj +
                                   Parameters.RealPatternSize.Width * ii]) / 128.0 - 1.0; // one is white, -one is black
                            }
                        });

                        // desired output vector

                        Parallel.For(0, nOutput, ii =>
                        {
                            targetOutputVector[ii] = -1.0;
                        });
                        targetOutputVector[Letters.IndexOf(label)] = 1.0;


                        // apply distortion map to inputVector.  It's not certain that this is needed or helpful.
                        // The second derivatives do NOT rely on the output of the neural net (i.e., because the 
                        // second derivative of the MSE function is exactly 1 (one), regardless of the actual output
                        // of the net).  However, since the backpropagated second derivatives rely on the outputs of
                        // each neuron, distortion of the pattern might reveal previously-unseen information about the
                        // nature of the Hessian.  But I am reluctant to give the full distortion, so I set the
                        // severityFactor to only 2/3 approx
                        GenerateDistortionMap(0.65);
                        ApplyDistortionMap(inputVector);
                        // forward calculate the neural network

                        Calculate(inputVector, (int)nInput, actualOutputVector, (int)nOutput, null);
                        // backpropagate the second derivatives
                        BackpropagateSecondDervatives(actualOutputVector, targetOutputVector, nOutput);
                        double ratio = (double)((double)kk / numPatternsSampled);
                        int n = (int)Math.Floor(ratio * 100);
                        if (parentControl != null)
                            parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 3, n });
                        //
                        if (token.IsCancellationRequested)
                        {
                            return;
                        }
                    }
                }
            }
            DivideHessianInformationBy((double)numPatternsSampled);
            s = " Caculation of Hessian...completed";
            if (parentControl != null)
                parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 2, s });
        }

        /////////////////////////
        /// <summary>
        /// 
        /// </summary>
        /// <param name="inputVector"></param>
        /// <param name="count"></param>
        /// <param name="outputVector"></param>
        /// <param name="oCount"></param>
        /// <param name="pNeuronOutputs"></param>
        /// <param name="bDistort"></param>

        /// <summary>
        /// 
        /// </summary>
        /// <param name="inputVector"></param>
        /// <param name="iCount"></param>
        /// <param name="targetOutputVector"></param>
        /// <param name="actualOutputVector"></param>
        /// <param name="oCount"></param>
        /// <param name="pMemorizedNeuronOutputs"></param>
        /// <param name="bDistort"></param>
        void BackpropagateNeuralNet(double[] inputVector, int iCount, double[] targetOutputVector,
                                       double[] actualOutputVector, int oCount,
                                       NeuronOutputs[] pMemorizedNeuronOutputs,
                                       bool bDistort, CancellationToken token)
        {
            // function to backpropagate through the neural net. 

            //ASSERT( (inputVector != NULL) && (targetOutputVector != NULL) && (actualOutputVector != NULL) );

            ///////////////////////////////////////////////////////////////////////
            //
            // CODE REVIEW NEEDED:
            //
            // It does not seem worthwhile to backpropagate an error that's very small.  "Small" needs to be defined
            // and for now, "small" is set to a fixed size of pattern error ErrP <= 0.10 * MSE, then there will
            // not be a backpropagation of the error.  The current MSE is updated from the neural net dialog CDlgNeuralNet

            bool bWorthwhileToBackpropagate;  /////// part of code review


            // local scope for capture of the neural net, only during the forward calculation step,
            // i.e., we release neural net for other threads after the forward calculation, and after we
            // have stored the outputs of each neuron, which are needed for the backpropagation step



            // determine if it's time to adjust the learning rate
          
            if (((Backprops % Parameters.AfterEveryNBackprops) == 0) && (Backprops != 0))
            {
                double eta = EtaLearningRate;
                eta *= Parameters.EtaDecay;
                if (eta < Parameters.MinimumEtaLearningRate)
                    eta = Parameters.MinimumEtaLearningRate;
                EtaLearningRate = eta;
            }


            // determine if it's time to adjust the Hessian (currently once per epoch)

            if ((IsNeedHessian != false) || ((Backprops % Parameters.AfterEveryNBackprops) == 0))
            {
                // adjust the Hessian.  This is a lengthy operation, since it must process approx 500 labels
                IsNeedHessian = false;
                CalculateHessian(token);

            }
            // increment counter for tracking number of backprops
            // determine if it's time to randomize the sequence of training patterns (currently once per epoch)
            // forward calculate through the neural net
            Task[] steps = new Task[2];
            steps[0] = new Task(() =>
            {
                CalculateNeuralNet(inputVector, iCount, actualOutputVector, oCount, pMemorizedNeuronOutputs, bDistort);
                // calculate error in the output of the neural net
                // note that this code duplicates that found in many other places, and it's probably sensible to 
                // define a (global/static ??) function for it
            });
            steps[0].Start();
            steps[1] = steps[0].ContinueWith((result) =>
            {
                double dMSE = 0.0;
                int nOutput=network.Layers.Last().NeuronCount;
                Parallel.For(0, nOutput, ii =>
                {
                    dMSE += (actualOutputVector[ii] - targetOutputVector[ii]) * (actualOutputVector[ii] - targetOutputVector[ii]);
                });
                dMSE /= 2.0;

                if (dMSE <= (0.10 * EstimatedCurrentMSE))
                {
                    bWorthwhileToBackpropagate = false;
                }
                else
                {
                    bWorthwhileToBackpropagate = true;
                }

                if ((bWorthwhileToBackpropagate != false) && (pMemorizedNeuronOutputs == null))
                {
                    // the caller has not provided a place to store neuron outputs, so we need to
                    // backpropagate now, while the neural net is still captured.  Otherwise, another thread
                    // might come along and call CalculateNeuralNet(), which would entirely change the neuron
                    // outputs and thereby inject errors into backpropagation 
                    Backpropagate(actualOutputVector, targetOutputVector, oCount, null);

                    // we're done, so return

                    return;
                }
                // if we have reached here, then the mutex for the neural net has been released for other 
                // threads.  The caller must have provided a place to store neuron outputs, which we can 
                // use to backpropagate, even if other threads call CalculateNeuralNet() and change the outputs
                // of the neurons

                if ((bWorthwhileToBackpropagate != false))
                {
                    Backpropagate(actualOutputVector, targetOutputVector, oCount, pMemorizedNeuronOutputs);
                }
            });
            steps[1].Wait();
        }
       
        public void BackpropagationThread( CancellationToken token)
        {
            // thread for backpropagation training of NN
            //
            // thread is "owned" by the doc, and accepts a pointer to the doc
            // continuously backpropagates until m_bThreadAbortFlag is set to TRUE  	
            int nInput = network.Layers.First().NeuronCount;
            int nOutput = network.Layers.Last().NeuronCount;
            double[] inputVector = new double[nInput];  // note: 29x29, not 28x28
            double[] targetOutputVector = new double[nOutput];
            double[] actualOutputVector = new double[nOutput];
            //
            Parallel.For(0, nInput, i =>
            {
                inputVector[i] = 0.0;
            });
            Parallel.For(0, nOutput, i =>
            {
                targetOutputVector[i] = 0.0;
                actualOutputVector[i] = 0.0;
            });
            //
            NeuronOutputs[] _memorizedNeuronOutputs = new NeuronOutputs[network.LayerCount];
            //prepare for training
            for (int epoch = 0; epoch < Parameters.Epochs; epoch++)
            {
                HPTime.Start();
                if (stopwatch.IsRunning)
                {
                    // Stop the timer; show the start and reset buttons.
                    stopwatch.Stop();
                }
               
                // Start the timer; show the stop and lap buttons.
                stopwatch.Reset();
                stopwatch.Start();
                
                String comment = "Randomize the patterns...";
                if (parentControl != null)
                    parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 2, comment });
                RandomizePatternSequence();
                if (epoch == Parameters.DistotionEpochs)
                {
                    Parameters.Distorted = false;
                }
                for (int iPattern = 0; iPattern < PatternCount; iPattern++)
                {

                    char label = new char();
                    byte[] grayLevels = new byte[Parameters.RealPatternSize.Width * Parameters.RealPatternSize.Height];
                    Task[] steps = new Task[3];

                    int pt = RandomTrainingPattern[iPattern];
                    grayLevels = patternsData[pt].Image;
                    label = patternsData[pt].Label;

                    if (network.IsUnknownOutput)
                    {
                        #region isUnknownOutput
                        // top row of inputVector is left as zero, left-most column is left as zero 

                        /*TODO: Check potentially-changing upper bound expression "NNDefinations.g_cImageSize" which is now called only *once*,
                        to ensure the new Parallel.For call matches behavior in the original for-loop
                        (where this upper bound expression had previously been evaluated at the start of *every* loop iteration).*/
                        try
                        {
                            Size smallestSize = new Size();
                            if (Parameters.DesignedPatternSize.Width >= Parameters.RealPatternSize.Width)
                            {
                                smallestSize.Width = Parameters.RealPatternSize.Width;
                            }
                            else
                            {
                                smallestSize.Width = Parameters.DesignedPatternSize.Width;
                            }
                            if (Parameters.DesignedPatternSize.Height >= Parameters.RealPatternSize.Height)
                            {

                                smallestSize.Height = Parameters.RealPatternSize.Height;
                            }
                            else
                            {

                                smallestSize.Height = Parameters.DesignedPatternSize.Height;
                            }
                            // convert unknown output
                            if (!Letters.Contains(label))
                            {
                                label = network.UnknownOuput;
                            }
                            steps[0] = Task.Factory.StartNew(() =>
                            {
                                //change byte input to double input;
                                //pad to designedpatternwidth*designedpatternheight, convert to double precision

                                Parallel.For(0, nInput, ii =>
                                {
                                    inputVector[ii] = (double)255.0 / 128.0 - 1.0;
                                    // one is white, -one is black
                                });
                                Parallel.For(0, smallestSize.Height, ii =>
                                {
                                    for (int jj = 0; jj < smallestSize.Width; ++jj)
                                    {
                                        inputVector[jj + Parameters.DesignedPatternSize.Width * ii] = (double)((int)(byte)grayLevels[jj +
                                           Parameters.RealPatternSize.Width * ii]) / 128.0 - 1.0; // one is white, -one is black
                                    }
                                });

                                // desired output vector

                                Parallel.For(0, nOutput, ii =>
                                {
                                    targetOutputVector[ii] = -1.0;
                                });
                                if (Letters.Contains(label))
                                {
                                    targetOutputVector[Letters.IndexOf(label)] = 1.0;
                                }
                                else
                                {
                                    targetOutputVector[nOutput - 1] = 1.0;
                                }
                            });
                            steps[1] = steps[0].ContinueWith((t) =>
                            {
                                BackpropagateNeuralNet(inputVector, nInput, targetOutputVector, actualOutputVector, nOutput,
                                    _memorizedNeuronOutputs, Parameters.Distorted, token);
                                // calculate error for this pattern and post it to the hwnd so it can calculate a running 
                                // estimate of MSE
                            });
                            steps[2] = steps[1].ContinueWith((t1) =>
                            {
                                double dMSE = 0.0;
                                Parallel.For(0, nOutput, ii =>
                                {
                                    dMSE += (actualOutputVector[ii] - targetOutputVector[ii]) * (actualOutputVector[ii] - targetOutputVector[ii]);
                                });
                                dMSE /= 2.0;
                                _dMSE += dMSE;
                                // determine the neural network's answer, and compare it to the actual answer.
                                // Post a message if the answer was incorrect, so the dialog can display mis-recognition
                                // statistics
                                _nn++;
                                int iBestIndex = 0;
                                double maxValue = -99.0;
                                for (int ii = 0; ii < nOutput; ii++)
                                {
                                    if (actualOutputVector[ii] > maxValue)
                                    {
                                        iBestIndex = ii;
                                        maxValue = actualOutputVector[ii];
                                    }
                                }
                                Char recognizedLabel;
                                if (label==network.UnknownOuput)
                                {
                                    if (iBestIndex != nOutput - 1)
                                    {
                                        MisrecognitionCount++;
                                        recognizedLabel = Letters[iBestIndex];
                                    }
                                    else
                                    {
                                        recognizedLabel = label;
                                    }
                                   
                                }
                                else
                                {
                                    if (iBestIndex != nOutput - 1)
                                    {
                                        if (iBestIndex != Letters.IndexOf(label))
                                        {
                                            MisrecognitionCount++;
                                            recognizedLabel = Letters[iBestIndex];
                                        }
                                        else
                                        {
                                            recognizedLabel = label;
                                        }
                                    }
                                    else
                                    {
                                        MisrecognitionCount++;
                                        recognizedLabel = network.UnknownOuput;
                                    }
                                }
                                double ratio = (double)((double)iPattern / PatternCount);
                                int n = (int)Math.Floor(ratio * 100);
                                object[] oj = new object[6];
                                oj[0] = n;
                                oj[1] = iPattern;
                                oj[2] = MisrecognitionCount;
                                oj[3] = _dMSE / iPattern;
                                oj[4] = recognizedLabel;
                                oj[5] = pt; //randomized pattern index
                                if (parentControl != null)
                                    parentControl.Invoke(parentControl.DelegateAddObjects, new Object[] { 99, oj });
                                oj = null;
                            });
                            steps[1].Wait();
                            if (token.IsCancellationRequested)
                            {

                                return;
                            }
                        }
                        catch (Exception ex)
                        {
                            throw;
                        }
#endregion
                    }
                    else
                    {
                        #region normal network
                        if (Letters.Contains(label))
                        {

                            // top row of inputVector is left as zero, left-most column is left as zero 

                            /*TODO: Check potentially-changing upper bound expression "NNDefinations.g_cImageSize" which is now called only *once*,
                            to ensure the new Parallel.For call matches behavior in the original for-loop
                            (where this upper bound expression had previously been evaluated at the start of *every* loop iteration).*/
                            Size smallestSize = new Size();
                            if (Parameters.DesignedPatternSize.Width >= Parameters.RealPatternSize.Width)
                            {
                                smallestSize.Width = Parameters.RealPatternSize.Width;
                            }
                            else
                            {
                                smallestSize.Width = Parameters.DesignedPatternSize.Width;
                            }
                            if (Parameters.DesignedPatternSize.Height >= Parameters.RealPatternSize.Height)
                            {

                                smallestSize.Height = Parameters.RealPatternSize.Height;
                            }
                            else
                            {

                                smallestSize.Height = Parameters.DesignedPatternSize.Height;
                            }

                            steps[0] = Task.Factory.StartNew(() =>
                            {
                                //change byte input to double input;
                                //pad to designedpatternwidth*designedpatternheight, convert to double precision

                                Parallel.For(0, nInput, ii =>
                                {
                                    inputVector[ii] = (double)255.0 / 128.0 - 1.0;
                                    // one is white, -one is black
                                });
                                Parallel.For(0, smallestSize.Height, ii =>
                                {
                                    for (int jj = 0; jj < smallestSize.Width; ++jj)
                                    {
                                        inputVector[jj + Parameters.DesignedPatternSize.Width * ii] = (double)((int)(byte)grayLevels[jj +
                                           Parameters.RealPatternSize.Width * ii]) / 128.0 - 1.0; // one is white, -one is black
                                    }
                                });

                                // desired output vector

                                Parallel.For(0, nOutput, ii =>
                                {
                                    targetOutputVector[ii] = -1.0;
                                });
                                targetOutputVector[Letters.IndexOf(label)] = 1.0;
                            });
                            steps[1] = steps[0].ContinueWith((t) =>
                            {
                                BackpropagateNeuralNet(inputVector, nInput, targetOutputVector, actualOutputVector, nOutput,
                                    _memorizedNeuronOutputs, Parameters.Distorted, token);
                                // calculate error for this pattern and post it to the hwnd so it can calculate a running 
                                // estimate of MSE
                            });
                            steps[2] = steps[1].ContinueWith((t1) =>
                            {
                                double dMSE = 0.0;
                                Parallel.For(0, nOutput, ii =>
                                {
                                    dMSE += (actualOutputVector[ii] - targetOutputVector[ii]) * (actualOutputVector[ii] - targetOutputVector[ii]);
                                });
                                dMSE /= 2.0;
                                _dMSE += dMSE;
                                // determine the neural network's answer, and compare it to the actual answer.
                                // Post a message if the answer was incorrect, so the dialog can display mis-recognition
                                // statistics
                                _nn++;
                                int iBestIndex = 0;
                                double maxValue = -99.0;
                                for (int ii = 0; ii < nOutput; ++ii)
                                {
                                    if (actualOutputVector[ii] > maxValue)
                                    {
                                        iBestIndex = ii;
                                        maxValue = actualOutputVector[ii];
                                    }
                                }
                                Char recognizedLabel;
                                if (iBestIndex != Letters.IndexOf(label))
                                {
                                    MisrecognitionCount++;
                                    recognizedLabel = Letters[iBestIndex];
                                }
                                else
                                {
                                    recognizedLabel = label;
                                }
                                double ratio = (double)((double)iPattern / PatternCount);
                                int n = (int)Math.Floor(ratio * 100);
                                object[] oj = new object[6];
                                oj[0] = n;
                                oj[1] = iPattern;
                                oj[2] = MisrecognitionCount;
                                oj[3] = _dMSE / iPattern;
                                oj[4] = recognizedLabel;
                                oj[5] = pt; //randomized pattern index
                                if (parentControl != null)
                                    parentControl.Invoke(parentControl.DelegateAddObjects, new Object[] { 99, oj });
                                oj = null;
                            });
                            steps[1].Wait();
                            if (token.IsCancellationRequested)
                            {

                                return;
                            }

                        }
                        #endregion
                    }
                    Backprops++;
                }// end of main "while not abort flag" loop
                HPTime.Stop();
                _dMSE /= PatternCount;
                if (stopwatch.IsRunning)
                {
                    // Stop the timer; show the start and reset buttons.
                    stopwatch.Stop();
                }
                // Get the elapsed time as a TimeSpan value.
                TimeSpan ts = stopwatch.Elapsed;

                // Format and display the TimeSpan value.
                String ElapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}",
                    ts.Hours, ts.Minutes, ts.Seconds,
                    ts.Milliseconds / 10);
                int minutes = (int)Math.Ceiling(HPTime.Duration / 60);

                TrainingResult ttResult = new TrainingResult();
                ttResult.CurrentMSE = _dMSE;
                ttResult.Distored = Parameters.Distorted;
                ttResult.Duration = ElapsedTime;
                ttResult.Epoch = (int)_iEpochsCompleted + 1;
                ttResult.MisPattern = (int)MisrecognitionCount;
                ttResult.Ratio = ((double)(PatternCount - MisrecognitionCount) / PatternCount) * 100;
                ttResult.EtaLearningRate = EtaLearningRate;
                Object[] oj1 = new Object[2];
                String s1 = String.Format("Completed Epochs:{0}, MisPatterns:{1}, MSE:{2}, Ex. time: {3} minutes, eta:{4} ",
                Convert.ToString(_iEpochsCompleted + 1), Convert.ToString(MisrecognitionCount), _dMSE.ToString(), minutes, EtaLearningRate.ToString());
                oj1[0] = (Object)ttResult;
                oj1[1] = s1;
                if (parentControl != null)
                    parentControl.Invoke(parentControl.DelegateAddObjects, new Object[] { 98, oj1 });
                MisrecognitionCount = 0;
                _iEpochsCompleted++;
                EstimatedCurrentMSE = _dMSE;
                _dMSE = 0;
                
            }
            //back propagation is completed
            String st = String.Format("BackPropagation is completed");
            if (parentControl != null)
            {
                parentControl.Invoke(parentControl.DelegateAddObject, new Object[] { 4, st });
            }
          
        }
   
    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

Vietdungiitb
Vietnam Maritime University
Vietnam Vietnam
No Biography provided

| Advertise | Privacy | Terms of Use | Mobile
Web01 | 2.8.1411023.1 | Last Updated 10 Apr 2013
Article Copyright 2013 by Vietdungiitb
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid