Click here to Skip to main content
11,410,365 members (58,443 online)
Click here to Skip to main content
Add your own
alternative version

Neural Networks on C#

, 19 Nov 2006 GPL3
The articles describes a C# library for neural network computations, and their application for several problem solving.
neuro_demo.zip
neuro_demo
Back Propagation
Approximation
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Approximation.exe
Data Samples
sample1.csv
sample2.csv
TimeSeries
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Data Samples
exponent.csv
growing sinusoid.csv
parabola.csv
sigmoid.csv
sinusoid.csv
TimeSeries.exe
XORProblem
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
XORProblem.exe
Simple
Delta Rule Learning
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Classifier.exe
Data Samples
and.csv
cube.csv
or.csv
sample1.csv
sample2.csv
One-Layer Perceptron Classifier
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Classifier.exe
Data Samples
sample1.csv
sample2.csv
Perceptron Classifier
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Classifier.exe
Data Samples
and.csv
cube.csv
or.csv
SOM
2DOrganizing
2DOrganizing.exe
AForge.dll
AForge.Neuro.dll
Color
AForge.dll
AForge.Neuro.dll
Color.exe
TSP
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
TSP.exe
neuro_src.zip
neuro_src
Docs
AForge.Core.chm
AForge.Neuro.chm
Release
AForge.Controls.dll
AForge.dll
AForge.Neuro.dll
Samples
Neuro
Back Propagation
Approximation
App.ico
Data Samples
sample1.csv
sample2.csv
TimeSeries
App.ico
Data Samples
exponent.csv
growing sinusoid.csv
parabola.csv
sigmoid.csv
sinusoid.csv
XORProblem
App.ico
Simple
Delta Rule Learning
App.ico
Data Samples
and.csv
cube.csv
or.csv
sample1.csv
sample2.csv
One-Layer Perceptron Classifier
App.ico
Data Samples
sample1.csv
sample2.csv
Perceptron Classifier
App.ico
Data Samples
and.csv
cube.csv
or.csv
SOM
2DOrganizing
App.ico
Color
App.ico
TSP
App.ico
Sources
Controls
Core
Neuro
Activation Functions
Images
sigmoid.bmp
sigmoid_bipolar.bmp
threshold.bmp
Layers
Learning
Networks
Neurons
// AForge Neural Net Library
//
// Copyright � Andrew Kirillov, 2005-2006
// andrew.kirillov@gmail.com
//

namespace AForge.Neuro.Learning
{
	using System;

	/// <summary>
	/// Back propagation learning algorithm
	/// </summary>
	/// 
	/// <remarks>The class implements back propagation learning algorithm,
	/// which is widely used for training multi-layer neural networks with
	/// continuous activation functions.</remarks>
	/// 
	public class BackPropagationLearning : ISupervisedLearning
	{
		// network to teach
		private ActivationNetwork network;
		// learning rate
		private double learningRate = 0.1;
		// momentum
		private double momentum = 0.0;

		// neuron's errors
		private double[][]		neuronErrors = null;
		// weight's updates
		private double[][][]	weightsUpdates = null;
		// threshold's updates
		private double[][]		thresholdsUpdates = null;


		/// <summary>
		/// Learning rate
		/// </summary>
		/// 
		/// <remarks>The value determines speed of learning. Default value equals to 0.1.</remarks>
		/// 
		public double LearningRate
		{
			get { return learningRate; }
			set
			{
				learningRate = Math.Max( 0.0, Math.Min( 1.0, value ) );
			}
		}

		/// <summary>
		/// Momentum
		/// </summary>
		/// 
		/// <remarks>The value determines the portion of previous weight's update
		/// to use on current iteration. Weight's update values are calculated on
		/// each iteration depending on neuron's error. The momentum specifies the amount
		/// of update to use from previous iteration and the amount of update
		/// to use from current iteration. If the value is equal to 0.1, for example,
		/// then 0.1 portion of previous update and 0.9 portion of current update are used
		/// to update weight's value.<br /><br />
		///	Default value equals to 0.0.</remarks>
		/// 
		public double Momentum
		{
			get { return momentum; }
			set
			{
				momentum = Math.Max( 0.0, Math.Min( 1.0, value ) );
			}
		}

		/// <summary>
		/// Initializes a new instance of the <see cref="BackPropagationLearning"/> class
		/// </summary>
		/// 
		/// <param name="network">Network to teach</param>
		/// 
		public BackPropagationLearning( ActivationNetwork network )
		{
			this.network = network;

			// create error and deltas arrays
			neuronErrors = new double[network.LayersCount][];
			weightsUpdates = new double[network.LayersCount][][];
			thresholdsUpdates = new double[network.LayersCount][];

			// initialize errors and deltas arrays for each layer
			for ( int i = 0, n = network.LayersCount; i < n; i++ )
			{
				Layer layer = network[i];

				neuronErrors[i] = new double[layer.NeuronsCount];
				weightsUpdates[i] = new double[layer.NeuronsCount][];
				thresholdsUpdates[i] = new double[layer.NeuronsCount];

				// for each neuron
				for ( int j = 0; j < layer.NeuronsCount; j++ )
				{
					weightsUpdates[i][j] = new double[layer.InputsCount];
				}
			}
		}

		/// <summary>
		/// Runs learning iteration
		/// </summary>
		/// 
		/// <param name="input">input vector</param>
		/// <param name="output">desired output vector</param>
		/// 
		/// <returns>Returns squared error of the last layer divided by 2</returns>
		/// 
		/// <remarks>Runs one learning iteration and updates neuron's
		/// weights.</remarks>
		///
		public double Run( double[] input, double[] output )
		{
			// compute the network's output
			network.Compute( input );

			// calculate network error
			double error = CalculateError( output );

			// calculate weights updates
			CalculateUpdates( input );

			// update the network
			UpdateNetwork( );

			return error;
		}

		/// <summary>
		/// Runs learning epoch
		/// </summary>
		/// 
		/// <param name="input">array of input vectors</param>
		/// <param name="output">array of output vectors</param>
		/// 
		/// <returns>Returns sum of squared errors of the last layer divided by 2</returns>
		/// 
		/// <remarks>Runs series of learning iterations - one iteration
		/// for each input sample. Updates neuron's weights after each sample
		/// presented.</remarks>
		/// 
		public double RunEpoch( double[][] input, double[][] output )
		{
			double error = 0.0;

			// run learning procedure for all samples
			for ( int i = 0, n = input.Length; i < n; i++ )
			{
				error += Run( input[i], output[i] );
			}

			// return summary error
			return error;
		}


		/// <summary>
		/// Calculates error values for all neurons of the network
		/// </summary>
		/// 
		/// <param name="desiredOutput">Desired output vector</param>
		/// 
		/// <returns>Returns summary squared error of the last layer divided by 2</returns>
		/// 
		private double CalculateError( double[] desiredOutput )
		{
			// current and the next layers
			ActivationLayer	layer, layerNext;
			// current and the next errors arrays
			double[] errors, errorsNext;
			// error values
			double error = 0, e, sum;
			// neuron's output value
			double output;
			// layers count
			int layersCount = network.LayersCount;

			// assume, that all neurons of the network have the same activation function
			IActivationFunction	function = network[0][0].ActivationFunction;

			// calculate error values for the last layer first
			layer	= network[layersCount - 1];
			errors	= neuronErrors[layersCount - 1];

			for ( int i = 0, n = layer.NeuronsCount; i < n; i++ )
			{
				output = layer[i].Output;
				// error of the neuron
				e = desiredOutput[i] - output;
				// error multiplied with activation function's derivative
				errors[i] = e * function.Derivative2( output );
				// squre the error and sum it
				error += ( e * e );
			}

			// calculate error values for other layers
			for ( int j = layersCount - 2; j >= 0; j-- )
			{
				layer		= network[j];
				layerNext	= network[j + 1];
				errors		= neuronErrors[j];
				errorsNext	= neuronErrors[j + 1];

				// for all neurons of the layer
				for ( int i = 0, n = layer.NeuronsCount; i < n; i++ )
				{
					sum = 0.0;
					// for all neurons of the next layer
					for ( int k = 0, m = layerNext.NeuronsCount; k < m; k++ )
					{
						sum += errorsNext[k] * layerNext[k][i];
					}
					errors[i] = sum * function.Derivative2( layer[i].Output );
				}
			}

			// return squared error of the last layer divided by 2
			return error / 2.0;
		}

		/// <summary>
		/// Calculate weights updates
		/// </summary>
		/// 
		/// <param name="input">Network's input vector</param>
		/// 
		private void CalculateUpdates( double[] input )
		{
			// current neuron
			ActivationNeuron	neuron;
			// current and previous layers
			ActivationLayer		layer, layerPrev;
			// layer's weights updates
			double[][]	layerWeightsUpdates;
			// layer's thresholds updates
			double[]	layerThresholdUpdates;
			// layer's error
			double[]	errors;
			// neuron's weights updates
			double[]	neuronWeightUpdates;
			// error value
			double		error;

			// 1 - calculate updates for the last layer fisrt
			layer = network[0];
			errors = neuronErrors[0];
			layerWeightsUpdates = weightsUpdates[0];
			layerThresholdUpdates = thresholdsUpdates[0];

			// for each neuron of the layer
			for ( int i = 0, n = layer.NeuronsCount; i < n; i++ )
			{
				neuron	= layer[i];
				error	= errors[i];
				neuronWeightUpdates	= layerWeightsUpdates[i];

				// for each weight of the neuron
				for ( int j = 0, m = neuron.InputsCount; j < m; j++ )
				{
					// calculate weight update
					neuronWeightUpdates[j] = learningRate * (
						momentum * neuronWeightUpdates[j] +
						( 1.0 - momentum ) * error * input[j]
						);
				}

				// calculate treshold update
				layerThresholdUpdates[i] = learningRate * (
					momentum * layerThresholdUpdates[i] +
					( 1.0 - momentum ) * error
					);
			}

			// 2 - for all other layers
			for ( int k = 1, l = network.LayersCount; k < l; k++ )
			{
				layerPrev			= network[k - 1];
				layer				= network[k];
				errors				= neuronErrors[k];
				layerWeightsUpdates	= weightsUpdates[k];
				layerThresholdUpdates = thresholdsUpdates[k];

				// for each neuron of the layer
				for ( int i = 0, n = layer.NeuronsCount; i < n; i++ )
				{
					neuron	= layer[i];
					error	= errors[i];
					neuronWeightUpdates	= layerWeightsUpdates[i];

					// for each synapse of the neuron
					for ( int j = 0, m = neuron.InputsCount; j < m; j++ )
					{
						// calculate weight update
						neuronWeightUpdates[j] = learningRate * (
							momentum * neuronWeightUpdates[j] +
							( 1.0 - momentum ) * error * layerPrev[j].Output
							);
					}

					// calculate treshold update
					layerThresholdUpdates[i] = learningRate * (
						momentum * layerThresholdUpdates[i] +
						( 1.0 - momentum ) * error
						);
				}
			}
		}

		/// <summary>
		/// Update network'sweights
		/// </summary>
		/// 
		private void UpdateNetwork( )
		{
			// current neuron
			ActivationNeuron	neuron;
			// current layer
			ActivationLayer		layer;
			// layer's weights updates
			double[][]	layerWeightsUpdates;
			// layer's thresholds updates
			double[]	layerThresholdUpdates;
			// neuron's weights updates
			double[]	neuronWeightUpdates;

			// for each layer of the network
			for ( int i = 0, n = network.LayersCount; i < n; i++ )
			{
				layer = network[i];
				layerWeightsUpdates = weightsUpdates[i];
				layerThresholdUpdates = thresholdsUpdates[i];

				// for each neuron of the layer
				for ( int j = 0, m = layer.NeuronsCount; j < m; j++ )
				{
					neuron = layer[j];
					neuronWeightUpdates = layerWeightsUpdates[j];

					// for each weight of the neuron
					for ( int k = 0, s = neuron.InputsCount; k < s; k++ )
					{
						// update weight
						neuron[k] += neuronWeightUpdates[k];
					}
					// update treshold
					neuron.Threshold += layerThresholdUpdates[j];
				}
			}
		}
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The GNU General Public License (GPLv3)

Share

About the Author

Andrew Kirillov
Software Developer (Senior) Cisco Systems
United Kingdom United Kingdom
Started software development at about 15 years old and it seems like now it lasts most part of my life. Fortunately did not spend too much time with Z80 and BK0010 and switched to 8086 and further. Similar with programming languages – luckily managed to get away from BASIC and Pascal to things like Assembler, C, C++ and then C#. Apart from daily programming for food, do it also for hobby, where mostly enjoy areas like Computer Vision, Robotics and AI. This led to some open source stuff like AForge.NET.

Going out of computers I am just a man loving his family, enjoying traveling, a bit of books, a bit of movies and a mixture of everything else. Always wanted to learn playing guitar, but it seems like 6 strings are much harder than few dozens of keyboard’s keys. Will keep progressing ...

| Advertise | Privacy | Terms of Use | Mobile
Web01 | 2.8.150414.5 | Last Updated 19 Nov 2006
Article Copyright 2006 by Andrew Kirillov
Everything else Copyright © CodeProject, 1999-2015
Layout: fixed | fluid