using System;
using SharpUtils;
using System.Collections;
using System.Text;
using System.Xml.Serialization;
using System.Xml;
namespace Neural_Net_Library
{
/// <summary>
/// back propagation output layer
/// </summary>
public class BackPropagationOutputNode : AdalineNode
{
private DebugLevel debugLevel;
private Logger log;
public BackPropagationOutputNode( Logger log ) : base( log )
{
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
/// <summary>
/// constructor
/// has three values items Nodevalue, learning rate and momentum
/// </summary>
/// <param name="log"></param>
/// <param name="dLearningRate">The learning rate for the network</param>
/// <param name="dMomentum">The momentum for the network</param>
public BackPropagationOutputNode( Logger log, double dLearningRate, double dMomentum ) : base( log, 3, 1 )
{
this.NodeValues[ Values.LearningRate ] = dLearningRate;
this.NodeValues[ Values.Momentum ] = dMomentum;
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
/// <summary>
/// override the transfer function
/// </summary>
/// <param name="dValue"></param>
/// <returns></returns>
protected override double TransferFunction( double dValue )
{
if( this.UseBias == true )
return 1.0/( 1+Math.Exp( dValue + this.GetBias.GetOriginalBias() ) );
else
return 1.0/( 1+Math.Exp( dValue ) ); /// sigma
}
protected override double TransferFunction( double dValue, double dBias )
{
return 1.0/( 1+Math.Exp( dValue + dBias ) );
}
/// <summary>
/// return node output error
/// </summary>
/// <returns></returns>
public virtual double ComputeError()
{
return ( ( double )this.NodeValues[ Values.NodeValue ] ) * ( 1.0-( double )this.NodeValues[ Values.NodeValue ] ) * ( ( ( double )this.NodeErrors[ Values.NodeError ] ) - ( ( double )this.NodeValues[ Values.NodeValue ] ) );
}
/// <summary>
/// learn function for backpropagtion network
/// </summary>
public override void Learn()
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Learn called for BackPropagation node ", ClassName );
}
double dDelta = 0.0;
this.NodeErrors[ Values.NodeError ] = ComputeError();
for( int i=0;i<this.InputLinks.Count; i++ )
{
dDelta = ( ( double )this.NodeValues[ Values.LearningRate ] ) * ( ( double )this.NodeErrors[ Values.NodeError ] ) * ( ( BasicLink )this.InputLinks[ i ] ).InputValue( Values.NodeValue );
( ( BackPropagationLink )this.InputLinks[ i ] ).UpdateWeight( dDelta );
}
}
public new string ClassName
{
get
{
return this.ToString();
}
}
/// <summary>
/// save the back propagation output node
/// </summary>
/// <param name="xmlWriter"></param>
public override void Save( XmlWriter xmlWriter )
{
xmlWriter.WriteStartElement( "BackPropagationOutputNode" );
base.Save( xmlWriter );
xmlWriter.WriteEndElement();
}
/// <summary>
/// load the back porpagation output node
/// </summary>
/// <param name="xmlReader"></param>
public override void Load( XmlReader xmlReader )
{
base.Load( xmlReader );
}
}
/// <summary>
/// back propagation middle layer
/// </summary>
public class BackPropagationMiddleNode : BackPropagationOutputNode
{
private DebugLevel debugLevel;
private Logger log;
public BackPropagationMiddleNode( Logger log ) : base( log )
{
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
/// <summary>
/// constructor
/// </summary>
/// <param name="log">Log file</param>
/// <param name="dLearningRate">Nodes learning rate</param>
/// <param name="dMomentum">Nodes momentum</param>
public BackPropagationMiddleNode( Logger log, double dLearningRate, double dMomentum ) : base( log, dLearningRate, dMomentum )
{
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
public new string ClassName
{
get
{
return this.ToString();
}
}
/// <summary>
/// compute the nodes error
/// </summary>
/// <returns></returns>
public override double ComputeError()
{
double dTotal = 0.0;
for( int i=0; i<this.OutputLinks.Count; i++ )
{
dTotal += ( ( BackPropagationLink )this.OutputLinks[ i ] ).WeightedOutputError( Values.NodeError );
}
return ( double )this.NodeValues[ Values.NodeValue ] * ( 1.0-( ( double )this.NodeValues[ Values.NodeValue ] ) ) * dTotal;
}
public override void Learn()
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Learn called for BackPropagation node ", ClassName );
}
double dDelta = 0.0;
this.NodeErrors[ Values.NodeError ] = ComputeError();
for( int i=0;i<this.InputLinks.Count; i++ )
{
dDelta = ( ( double )this.NodeValues[ Values.LearningRate ] ) * ( ( double )this.NodeErrors[ Values.NodeError ] ) * ( ( BasicLink )this.InputLinks[ i ] ).InputValue( Values.NodeValue );
( ( BackPropagationLink )this.InputLinks[ i ] ).UpdateWeight( dDelta );
}
}
/// <summary>
/// save the current node
/// </summary>
/// <param name="xmlWriter"></param>
public override void Save( XmlWriter xmlWriter )
{
xmlWriter.WriteStartElement( "BackPropagationMiddleNode" );
base.Save( xmlWriter );
xmlWriter.WriteEndElement();
}
/// <summary>
/// load a saved node
/// </summary>
/// <param name="xmlReader"></param>
public override void Load( XmlReader xmlReader )
{
base.Load( xmlReader );
}
}
/// <summary>
/// backp[ropagation link class
/// </summary>
public class BackPropagationLink : BasicLink
{
private DebugLevel debugLevel;
private Logger log;
/// <summary>
/// constructor calls BasicLink ( Logger log, int nCount )
/// As it requires 2 default members the weight and the delta
/// </summary>
/// <param name="log"></param>
public BackPropagationLink( Logger log ) : base( log, 2 )
{
this.arrayLinkValues[ Values.Weight ] = Values.Random( -1.0, 1.0 );
this.arrayLinkValues[ Values.Delta ] = 0.0;
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
/// <summary>
/// save the back propagation link
/// </summary>
/// <param name="xmlWriter"></param>
public override void Save( XmlWriter xmlWriter )
{
xmlWriter.WriteStartElement( "BackPropagationLink" );
base.Save( xmlWriter );
xmlWriter.WriteEndElement();
}
/// <summary>
/// load the back propagation link
/// </summary>
/// <param name="xmlReader"></param>
public override void Load( XmlReader xmlReader )
{
base.Load( xmlReader );
}
public new string ClassName
{
get
{
return this.ToString();
}
}
public override void UpdateWeight( double dNewValue )
{
/// get the current momentum
double dMomentum = this.OutputNode.GetValue( Values.Momentum );
/// update the weight with the current chande and a percentage of the last change
this.arrayLinkValues[ Values.Weight ] = ( double )this.arrayLinkValues[ Values.Weight ] + dNewValue + ( dMomentum * ( double )this.arrayLinkValues[ Values.Delta ] );
/// store the new value as passed
this.arrayLinkValues[ Values.Delta ] = dNewValue;
}
}
/// <summary>
/// backpropagation network class
/// </summary>
public class BackPropagationNetwork : BasicNetwork
{
private int nNumLayers;
private int nFirstMiddleNode;
private int nFirstOutputNode;
private double dMomentumTerm;
private ArrayList arrayLayers;
private int nNumberOfNodes = 0;
private int nNumberOfLinks = 0;
private DebugLevel debugLevel;
private Logger log;
public int NumberOfLayers
{
get
{
return nNumLayers;
}
set
{
nNumLayers = value;
}
}
public int FirstMiddleNode
{
get
{
return nFirstMiddleNode;
}
}
public int FirstOutputNode
{
get
{
return nFirstOutputNode;
}
}
public double MomentumTerm
{
get
{
return dMomentumTerm;
}
}
public ArrayList Layers
{
get
{
return arrayLayers;
}
set
{
arrayLayers = value;
}
}
/// <summary>
/// add a new layer to the network passing the number of nodes as an int
/// </summary>
public int AddLayer
{
set
{
arrayLayers.Add( value );
}
}
public BackPropagationNetwork( Logger log, int nNodesCount, int nLinksCount ) : base( log, nNodesCount, nLinksCount )
{
arrayLayers = new ArrayList();
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
public BackPropagationNetwork( Logger log, double dLearningRate, double dMomentum, int nLayers ) : base( log )
{
arrayLayers = new ArrayList( nLayers );
nNumLayers = nLayers;
this.LearningRate = dLearningRate;
this.dMomentumTerm = dMomentum;
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
}
/// <summary>
/// quick constructor for a three layer network
/// TODO Write a version of this using variable parameters ( params )
/// </summary>
/// <param name="log">Logging object</param>
/// <param name="dLearningRate">learning rate for the network</param>
/// <param name="dMomentum">momentum of the network</param>
/// <param name="nLayerOne">number of nodes in layer one</param>
/// <param name="nLayerTwo">number of nodes in layer two</param>
/// <param name="nLayerThree">number of nodes in layer three</param>
public BackPropagationNetwork( Logger log, double dLearningRate, double dMomentum, int nLayerOne, int nLayerTwo, int nLayerThree ) : base( log )
{
debugLevel = new DebugLevel( DebugLevel.currentLevel );
this.log = log;
arrayLayers = new ArrayList( 3 );
arrayLayers.Add( nLayerOne );
arrayLayers.Add( nLayerTwo );
arrayLayers.Add( nLayerThree );
nNumLayers = 3;
this.LearningRate = dLearningRate;
this.dMomentumTerm = dMomentum;
CreateNetwork();
}
/// <summary>
/// create the back propagation network
/// </summary>
protected override void CreateNetwork()
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Create Network called for the backpropagation network ", ClassName );
}
/// work out the number of nodes and links
for( int i=0; i<arrayLayers.Count; i++ )
{
nNumberOfNodes += ( int )arrayLayers[ i ];
}
/// number of links equals the a link to each node in the preceeding layer
for( int i=1; i<arrayLayers.Count; i++ )
{
nNumberOfLinks += ( ( int )arrayLayers[ i-1 ] * ( int )arrayLayers[ i ] );
}
/// fill out the node arrays
nFirstMiddleNode = 0;
for( int i=0; i<nNumLayers; i++ )
{
/// input layer
if( i==0 )
{
for( int n=0; n<( int )arrayLayers[ i ]; n++ )
{
this.AddNode( new BasicNode( log ) );
}
}
/// output layer
else if( i+1 == nNumLayers ) /// numlayers not 0 based
{
nFirstOutputNode = this.Nodes.Count;
for( int n=0; n<( int )arrayLayers[ i ]; n++ )
{
this.AddNode( new BackPropagationOutputNode( log, this.LearningRate, this.dMomentumTerm ) );
}
}
/// middle layer (s)
else
{
nFirstMiddleNode = this.Nodes.Count;
for( int n=0; n<( int )arrayLayers[ i ]; n++ )
{
this.AddNode( new BackPropagationMiddleNode( log, this.LearningRate, this.dMomentumTerm ) );
}
}
}
/// create the links
for( int i=0; i<nNumberOfLinks; i++ )
{
this.AddLink( new BackPropagationLink( log ) );
}
/// now do all the connections
int nLayerOne = 0;
int nLayerTwo = nFirstMiddleNode;
int nLinkNumber = 0;
for( int i=0; i<nNumLayers-1; i++ )
{
/// outer layer ( starts with input layer )
for( int n=0; n<( int )arrayLayers[ i ]; n++ )
{
/// next inner layer to link to the outer layer
for( int k=0; k<( ( int )arrayLayers[ i + 1 ] ); k++ )
{
( ( BasicNode )this.Nodes[ nLayerOne + n ] ).CreateLink( ( BasicNode )this.Nodes[ nLayerTwo + k ], ( BasicLink )this.Links[ nLinkNumber ] );
nLinkNumber++;
}
}
nLayerOne = nLayerTwo;
nLayerTwo += ( int )Layers[ i + 1 ];
}
}
/// <summary>
/// publicly accessible create network so code can be built form scratch if required
/// assumes layers have been added through the AddLayer function
/// requires the number of layers to differentiate it from protected version
/// Call with CreateNetwork( backpropagationNetworkVariable.Layers );
/// </summary>
public void CreateNetwork( int nNumberLayers )
{
CreateNetwork();
}
/// <summary>
/// get the output error for the specified node
/// </summary>
/// <returns></returns>
public virtual double OutputError( int nID )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Getting the Output Error at " + nID.ToString() + " from the Backpropagation network", ClassName );
}
if( this.Nodes.Count < ( nID + nFirstOutputNode ) )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.WarningsAndErrors ) == true )
{
log.Log( DebugLevelSet.WarningsAndErrors, "Warning the array count is less than the index you are using to access it, returning 0.0 ", ClassName );
}
return 0.0;
}
return this.GetNodeAt( nID + nFirstOutputNode ).GetError( Values.NodeError );
}
/// <summary>
/// set the output error at the passed index
/// </summary>
/// <param name="dNewValue"></param>
/// <param name="nID"></param>
public virtual void SetOutputError( int nID, double dNewValue )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Setting the output error for output node " + nID.ToString() + " from " + this.GetNodeAt( nID + nFirstOutputNode ).GetError( Values.NodeError ).ToString() + " to " + this.GetNodeAt( nID + nFirstOutputNode ).GetError( Values.NodeError ), ClassName );
}
if( this.Nodes.Count < ( nID + nFirstOutputNode ) )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.WarningsAndErrors ) == true )
{
log.Log( DebugLevelSet.WarningsAndErrors, "Warning the array count is less that the index you are using to access it, quiting set node error in Backpropagation network", ClassName );
}
return;
}
this.GetNodeAt( nID + nFirstOutputNode ).SetError( Values.NodeError, dNewValue );
}
/// <summary>
/// Set the error values in the output nodes using the passed pattern
/// </summary>
/// <param name="pattern"></param>
public virtual void SetOutputError( Pattern pattern )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Set output error with pattern called for backpropagation network", ClassName );
}
for( int i=0; i<( ( int )this.Layers[ nNumLayers-1 ] ); i++ )
{
this.GetNodeAt( i + nFirstOutputNode ).SetError( Values.NodeError, pattern.OutputValue( i ) );
}
}
/// <summary>
/// get the value for the output node at the given index
/// </summary>
/// <param name="nID"></param>
/// <returns></returns>
public virtual double GetOutputValue( int nID )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Get output value called for backpropagation network", ClassName );
}
if( this.Nodes.Count < ( nID + nFirstOutputNode ) )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.WarningsAndErrors ) == true )
{
log.Log( DebugLevelSet.WarningsAndErrors, "Warning the array count is less than the index you are using to access it, returning 0.0", ClassName );
}
return 0.0;
}
return this.GetNodeAt( nID + nFirstOutputNode ).GetValue( Values.NodeValue );
}
/// <summary>
/// This function returns the calculated output value for the backprop network
/// that is the absolute output value - pattern outvalue and return the absolute result
/// </summary>
/// <param name="nID"></param>
/// <returns></returns>
public virtual double GetCalculatedOutputValue( int nID, double dPatternOutputValue )
{
double dTest = ( Math.Abs( GetOutputValue( nID ) ) - dPatternOutputValue );
return Math.Abs( dTest );
}
/// <summary>
/// save the current network
/// </summary>
/// <param name="xmlWriter"></param>
public override void Save( XmlWriter xmlWriter )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Save Node Links called for Back Propagation network ", ClassName );
}
xmlWriter.WriteStartElement( "BackPropagationNetwork" );
xmlWriter.WriteElementString( "NumberOfLayers", nNumLayers.ToString() );
xmlWriter.WriteElementString( "FirstMiddleNode", nFirstMiddleNode.ToString() );
xmlWriter.WriteElementString( "FirstOutputNode", nFirstOutputNode.ToString() );
xmlWriter.WriteElementString( "Momentum", dMomentumTerm.ToString() );
xmlWriter.WriteStartElement( "Layers" );
for( int i=0; i<nNumLayers; i++ )
{
xmlWriter.WriteElementString( "Layer" + i.ToString(), ( ( int )arrayLayers[ i ] ).ToString() );
}
xmlWriter.WriteEndElement();
for( int i=0; i<nFirstMiddleNode; i++ )
{
this.GetNodeAt( i ).Save( xmlWriter );
}
for( int i=nFirstMiddleNode; i<nFirstOutputNode; i++ )
{
( ( BackPropagationMiddleNode )this.Nodes[ i ] ).Save( xmlWriter );
}
for( int i=nFirstOutputNode; i<this.Nodes.Count; i++ )
{
( ( BackPropagationOutputNode )this.Nodes[ i ] ).Save( xmlWriter );
}
for( int i=0; i<this.Links.Count; i++ )
{
( ( BackPropagationLink )this.Links[ i ] ).Save( xmlWriter );
}
xmlWriter.WriteEndElement();
}
/// <summary>
/// load the network from a file
/// </summary>
/// <param name="xmlReader"></param>
public override void Load( XmlReader xmlReader )
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Load Node links called for Backpropagation network", ClassName );
}
bool bBreak = false;
this.Nodes.Clear();
this.Links.Clear();
while( xmlReader.Name != "BackPropagationNetwork" )
{
xmlReader.Read();
}
while( xmlReader.Name != "NumberOfLayers" )
{
xmlReader.Read();
}
xmlReader.Read();
NumberOfLayers = Int32.Parse( xmlReader.Value );
while( xmlReader.Name != "FirstMiddleNode" )
{
xmlReader.Read();
}
xmlReader.Read();
nFirstMiddleNode = Int32.Parse( xmlReader.Value );
while( xmlReader.Name != "FirstOutputNode" )
{
xmlReader.Read();
}
xmlReader.Read();
nFirstOutputNode = Int32.Parse( xmlReader.Value );
while( xmlReader.Name != "Momentum" )
{
xmlReader.Read();
}
xmlReader.Read();
dMomentumTerm = Double.Parse( xmlReader.Value );
while( xmlReader.Name != "Layers" )
{
xmlReader.Read();
}
for( int i=0; i<NumberOfLayers; i++ )
{
while( xmlReader.Name != "Layer" + i.ToString() )
{
xmlReader.Read();
}
xmlReader.Read();
arrayLayers.Add( xmlReader.Value );
}
/// can be any number of basic nodes not just one
for( ;; )
{
xmlReader.Read();
switch( xmlReader.NodeType )
{
case XmlNodeType.Element:
{
switch( xmlReader.Name )
{
case "BasicNode":
{
BasicNode temp = new BasicNode( log );
temp.Load( xmlReader );
this.Nodes.Add( temp );
} break;
case "BackPropagationMiddleNode":
{
BackPropagationMiddleNode temp = new BackPropagationMiddleNode( log );
temp.Load( xmlReader );
this.Nodes.Add( temp );
} break;
case "BackPropagationOutputNode":
{
BackPropagationOutputNode temp = new BackPropagationOutputNode( log );
temp.Load( xmlReader );
this.Nodes.Add( temp );
} break;
case "BackPropagationLink": bBreak = true; break;
}
} break;
}
if( bBreak == true )
break;
}
/// should now be on Basic Link
bBreak = false;
for( ;; )
{
switch( xmlReader.NodeType )
{
case XmlNodeType.Element:
{
switch( xmlReader.Name )
{
case "BackPropagationLink":
{
BackPropagationLink temp = new BackPropagationLink( log );
temp.Load( xmlReader );
this.Links.Add( temp );
} break;
}
} break;
case XmlNodeType.EndElement:
{
switch( xmlReader.Name )
{
case "BackPropagationNetwork": bBreak = true; break;
} break;
}
}
if( bBreak == true )
break;
xmlReader.Read();
}
/// now do all the connections
int nLayerOne = 0;
int nLayerTwo = nFirstMiddleNode;
int nLinkNumber = 0;
for( int i=0; i<nNumLayers-1; i++ )
{
/// outer layer ( starts with input layer )
for( int n=0; n<( int )arrayLayers[ i ]; n++ )
{
/// next inner layer to link to the outer layer
for( int k=0; k<( ( int )arrayLayers[ i + 1 ] ); k++ )
{
( ( BasicNode )this.Nodes[ nLayerOne + n ] ).CreateLink( ( BasicNode )this.Nodes[ nLayerTwo + k ], ( BasicLink )this.Links[ nLinkNumber ] );
nLinkNumber++;
}
}
nLayerOne = nLayerTwo;
nLayerTwo += ( int )Layers[ i + 1 ];
}
}
/// <summary>
/// run the nodes in the network
/// </summary>
public virtual void Run()
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Run called for Back propagation network", ClassName );
}
for( int i=nFirstMiddleNode; i<this.Nodes.Count; i++ )
{
( ( AdalineNode )this.Nodes[ i ] ).Run( Values.NodeValue );
}
}
/// <summary>
/// call learn on the networks nodes
/// </summary>
public virtual void Learn()
{
if( debugLevel.TestDebugLevel( DebugLevelSet.Progress ) == true )
{
log.Log( DebugLevelSet.Progress, "Learn called for Backpropagation network", ClassName );
}
/// note the learn runs backwards from the output nodes through the middle layers
for( int i=this.Nodes.Count-1; i>=nFirstMiddleNode; i-- )
{
( ( AdalineNode )this.Nodes[ i ] ).Learn();
}
}
public new string ClassName
{
get
{
return this.ToString();
}
}
}
}