Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Neural Network for Recognition of Handwritten Digits

, 5 Dec 2006
A convolutional neural network achieves 99.26% accuracy on a modified NIST database of hand-written digits.
// NeuralNetwork.h: interface for the NeuralNetwork class.
//
//////////////////////////////////////////////////////////////////////

#if !defined(AFX_NEURALNETWORK_H__186C10A1_9662_4C1C_B5CB_21F2F361D268__INCLUDED_)
#define AFX_NEURALNETWORK_H__186C10A1_9662_4C1C_B5CB_21F2F361D268__INCLUDED_

#if _MSC_VER > 1000
#pragma once
#endif // _MSC_VER > 1000

#include <math.h>
#include <vector>

using namespace std;

#define SIGMOID(x) (1.7159*tanh(0.66666667*x))
#define DSIGMOID(S) (0.66666667/1.7159*(1.7159+(S))*(1.7159-(S)))  // derivative of the sigmoid as a function of the sigmoid's output


// forward declarations

class NNLayer;
class NNWeight;
class NNNeuron;
class NNConnection;




// helpful typedef's

typedef std::vector< NNLayer* >  VectorLayers;
typedef std::vector< NNWeight* >  VectorWeights;
typedef std::vector< NNNeuron* >  VectorNeurons;
typedef std::vector< NNConnection > VectorConnections;
typedef std::basic_string<TCHAR>  tstring;


class NeuralNetwork  
{
public:
	volatile double m_etaLearningRatePrevious;
	volatile double m_etaLearningRate;

	volatile UINT m_cBackprops;  // counter used in connection with Weight sanity check
	void PeriodicWeightSanityCheck();

	void Calculate(double* inputVector, UINT count, 
		double* outputVector = NULL, UINT oCount = 0,
		std::vector< std::vector< double > >* pNeuronOutputs = NULL );

	void Backpropagate(double *actualOutput, double *desiredOutput, UINT count,
		std::vector< std::vector< double > >* pMemorizedNeuronOutputs );

	void EraseHessianInformation();
	void DivideHessianInformationBy( double divisor );
	void BackpropagateSecondDervatives( double* actualOutputVector, double* targetOutputVector, UINT count );

	void Serialize(CArchive &ar);

	NeuralNetwork();
	virtual ~NeuralNetwork();
	void Initialize();

	VectorLayers m_Layers;


};





class NNLayer
{
public:

	void PeriodicWeightSanityCheck();  // check if weights are "reasonable"
	void Calculate();
	void Backpropagate( std::vector< double >& dErr_wrt_dXn /* in */, 
		std::vector< double >& dErr_wrt_dXnm1 /* out */, 
		std::vector< double >* thisLayerOutput,  // memorized values of this layer's output
		std::vector< double >* prevLayerOutput,  // memorized values of previous layer's output
		double etaLearningRate );

	void EraseHessianInformation();
	void DivideHessianInformationBy( double divisor );
	void BackpropagateSecondDerivatives( std::vector< double >& dErr_wrt_dXn /* in */, 
		std::vector< double >& dErr_wrt_dXnm1 /* out */);

	void Serialize(CArchive& ar );

	NNLayer();
	NNLayer( LPCTSTR str, NNLayer* pPrev = NULL );
	virtual ~NNLayer();


	VectorWeights m_Weights;
	VectorNeurons m_Neurons;

	tstring label;
	NNLayer* m_pPrevLayer;

	bool m_bFloatingPointWarning;  // flag for one-time warning (per layer) about potential floating point overflow

protected:
	void Initialize();

};




class NNConnection
{
public: 
	NNConnection(UINT neuron = ULONG_MAX, UINT weight = ULONG_MAX):NeuronIndex( neuron ), WeightIndex( weight ) {};
	virtual ~NNConnection() {};
	UINT NeuronIndex, WeightIndex;
};




class NNWeight
{
public:
	NNWeight();
	NNWeight( LPCTSTR str, double val = 0.0 );
	virtual ~NNWeight();

	tstring label;
	double value;
	double diagHessian;


protected:
	void Initialize();

};


class NNNeuron
{
public:
	NNNeuron();
	NNNeuron( LPCTSTR str );
	virtual ~NNNeuron();

	void AddConnection( UINT iNeuron, UINT iWeight );
	void AddConnection( NNConnection const & conn );


	tstring label;
	double output;

	VectorConnections m_Connections;

///	VectorWeights m_Weights;
///	VectorNeurons m_Neurons;
	
protected:
	void Initialize();

};


#endif // !defined(AFX_NEURALNETWORK_H__186C10A1_9662_4C1C_B5CB_21F2F361D268__INCLUDED_)

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here

Share

About the Author

Mike O'Neill

United States United States
Mike O'Neill is a patent attorney in Southern California, where he specializes in computer and software-related patents. He programs as a hobby, and in a vain attempt to keep up with and understand the technology of his clients.

| Advertise | Privacy | Mobile
Web02 | 2.8.140827.1 | Last Updated 5 Dec 2006
Article Copyright 2006 by Mike O'Neill
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid