Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

CNeuralNetwork: Make Your Neural Network Learn Faster

, 12 Aug 2009 CPOL
An article on making neural network learn faster
Neural_Network_VS2008.zip
Neural Network_VS
Neural Network
Neural Network-Demo.exe
Neural Network.vcproj.RIS-808E3E7FF65.exeskeleton.user
SPECT.train
#include "stdafx.h"
#include "Neural Network.h"

CNeuralNetwork::CNeuralNetwork()
{
}

CNeuralNetwork::~CNeuralNetwork()
{
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//PRIVATE MEMBERS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

float CNeuralNetwork::rand_float_range(float a, float b)
{
	return ((b-a)*((float)rand()/RAND_MAX))+a;
}

float CNeuralNetwork::sigmoid(float f_net)
{
	return (float) ((2/(1+exp(-1*m_slope*f_net)))-1);
}

float CNeuralNetwork::sigmoid_derivative(float result)
{
	return (float) 0.5*(1-pow(result,2));
}

float CNeuralNetwork::get_norm_of_weight(int layer_num, int neuron_num)
{
	float ret = 0;
	for(int k=0;k<m_neuron_num[layer_num-1];k++){
        ret = ret + pow(m_weight[layer_num][neuron_num][k],2);
	}
	return ret;
}

void CNeuralNetwork::initialize_weights()
{
	// METHOD 1
	if (m_method == HARD_RANDOM){
		for(int i=1;i<m_layer_num;i++)
			for(int j=0;j<m_neuron_num[i];j++)
				for(int k=0;k<m_neuron_num[i-1];k++)
					m_weight[i][j][k]=rand_float_range(-m_init_val, m_init_val);
	}

	// METHOD 2
	else if (m_method == RANDOM){
		float range = sqrt(m_learning_rate / m_neuron_num[0]);

		for(int i=1;i<m_layer_num;i++)
			for(int j=0;j<m_neuron_num[i];j++)
				for(int k=0;k<m_neuron_num[i-1];k++)
					m_weight[i][j][k]=rand_float_range(-range, range);
	}

	// METHOD 3
	else if (m_method == NGUYEN){
		for(int i=1;i<m_layer_num;i++)
			for(int j=0;j<m_neuron_num[i];j++)
				for(int k=0;k<m_neuron_num[i-1];k++)
					m_weight[i][j][k]=rand_float_range(-1, 1);

		for(int i=1;i<m_layer_num;i++){
			float beta = 0.7 * pow((float) m_neuron_num[i], (float) 1/m_neuron_num[0]);
			for(int j=0;j<m_neuron_num[i];j++)
				for(int k=0;k<m_neuron_num[i-1];k++)
					m_weight[i][j][k]=beta * m_weight[i][j][k] / get_norm_of_weight(i,j);
		}
	}
}

// MEAN SQUARED ERROR
float CNeuralNetwork::get_mse_error()
{
	float result = 0.0F;
	int number_of_output_nodes = m_neuron_num[m_layer_num - 1];
	for( int i = 0; i < number_of_output_nodes; i++)
		result = result + pow((m_desired_output[i] - m_node_output[m_layer_num - 1][i]), 2);
	return result / 2;
}

void CNeuralNetwork::parse_data(string data_seq, int parsing_direction)
{
	//if seq == INPUT_FIRST --> string format: INPUT OUTPUT
	//if seq == OUTPUT_FIRST --> string format: OUTPUT INPUT

	size_t found = 0;
	size_t comma = 0;
	size_t start = 0;
	int i = 0;

	size_t length = data_seq.length();

	while(found < length ){
		//clean up white spaces
		while(data_seq.at(found) == ' ' || data_seq.at(found) == ',')
			found++;

		start = found;
		found = data_seq.find(' ', found + 1);
		comma = data_seq.find(',', start + 1);
		if (found > comma)
			found = comma;

		char buff[64];
		data_seq.copy(buff, found - start, start);

		if (parsing_direction == INPUT_FIRST){
			if (i < m_neuron_num[0])
				m_current_input[i] = atof(buff);
			else
				m_desired_output[i - m_neuron_num[0]] = atof(buff);
		}
		else{
			if (i < m_neuron_num[m_layer_num - 1])
				m_desired_output[i] = atof(buff);
			else
				m_current_input[i - m_neuron_num[m_layer_num - 1]] = atof(buff);
		}

		i++;
	}
}

void CNeuralNetwork::calculate_outputs()
{
	float f_net;
    int number_of_weights;
    for(int i=0;i<m_layer_num;i++)
        for(int j=0;j<m_neuron_num[i];j++)
        {
			m_node_output_prev[i][j] = m_node_output[i][j]; // store previous values
            f_net = 0.0F;
            if(i == 0)
				number_of_weights = 1;
            else
				number_of_weights = m_neuron_num[i-1];

            for(int k=0;k<number_of_weights;k++)
                if(i == 0)
                    f_net=m_current_input[j];
                else
                    f_net=f_net+m_node_output[i-1][k]*m_weight[i][j][k];
			if (i == 0)
				m_node_output[i][j] = f_net;	// for input layer we are using unity function
			else
				m_node_output[i][j]=sigmoid(f_net);
        }
}

void CNeuralNetwork::calculate_weights()
{
	for(int i=1;i<m_layer_num;i++){
		for(int j=0;j<m_neuron_num[i];j++){
            for(int k=0;k<m_neuron_num[i-1];k++)
            {
				float delta = m_learning_rate * m_error[i][j] * m_node_output[i-1][k];
				float delta_prev = m_learning_rate * m_error_prev[i][j] * m_node_output_prev[i-1][k];
				m_weight[i][j][k] = (float) m_weight[i][j][k] + delta + m_momentum * delta_prev;
            }
		}
	}
}

void CNeuralNetwork::calculate_errors()
{
    float sum = 0.0F;
	int number_of_output_nodes = m_neuron_num[m_layer_num - 1];

	for( int i = 0; i< number_of_output_nodes; i++){
		m_error_prev[m_layer_num - 1][i] = m_error[m_layer_num - 1][i]; // store previous values
        m_error[m_layer_num - 1][i] = (float) ((m_desired_output[i] - m_node_output[m_layer_num-1][i]) * sigmoid_derivative(m_node_output[m_layer_num-1][i]));
	}

	for(int i = m_layer_num-2; i>=0; i--){
        for( int j=0;j<m_neuron_num[i];j++)
        {
            sum = 0.0F;
            for( int k = 0; k < m_neuron_num[i+1]; k++)
                sum = sum + m_error[i+1][k] * m_weight[i+1][k][j];
			m_error_prev[i][j] = m_error[i][j]; // store previous values
            m_error[i][j] = (float) (sigmoid_derivative(m_node_output[i][j]) * sum);
        }
	}
}


void CNeuralNetwork::generate_report()
{
    ofstream log ("result.log");

    log << "number of layers : " <<  m_layer_num << endl;
    for (int i = 0; i < m_layer_num; i++)
        log << "neuron numbers in layer " << i << ": " << m_neuron_num[i] << endl;

    log << endl << "number of epoch needed: " << m_epoch << endl;
    log << "final average mse: " << m_average_error << endl << endl;

    log << "weights value: " << endl;
    log << "==============" << endl;

    for(int i=1;i<m_layer_num;i++){
        log << endl << "layer: "  << i << endl;
        for(int j=0;j<m_neuron_num[i];j++){
			for(int k=0;k<m_neuron_num[i-1];k++){
                log << "w" << k << j << ": " << m_weight[i][j][k] << endl;
			}
        }
    }

    log.close();
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//PUBLIC MEMBERS
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

void CNeuralNetwork::ann_create_network(unsigned int input_num, unsigned int output_num, unsigned int hidden_layer_num, ...)
{
	m_layer_num = hidden_layer_num + 2; // preserve extra spaces for input and output layers

	m_current_input  = new float[input_num];
	m_desired_output = new float[output_num];
	m_neuron_num	 = new int[m_layer_num];

	// Get number of neuron for each hidden layer
	va_list neuron_num;
	va_start(neuron_num, hidden_layer_num);
	for (unsigned int i = 0; i < hidden_layer_num; i++)
		m_neuron_num[i+1] = va_arg(neuron_num, unsigned int);
	va_end(neuron_num);

	// For input and output layer
	m_neuron_num[0] = input_num;
	m_neuron_num[m_layer_num - 1] = output_num;

	/////////////////////////////////////////////////////////
	// --- CREATE ALL DYNAMIC MULTIDIMENSIONAL ARRAY HERE ---

	// m_weight is 3-D array
	// m_weight[layer][number of neuron on previous layer][number of neuron on current layer]
	m_weight = new float**[m_layer_num];
	if (m_weight != NULL){
		for (int i = 1; i < m_layer_num; i++){
			*(m_weight+i) = new float*[m_neuron_num[i]];
			for (int j = 0; j < m_neuron_num[i]; j++)
				*(*(m_weight+i)+j) = new float[m_neuron_num[i-1]];
		}
	}

	// 2-D array
	// m_array_name[layer][number of neuron on current layer]
	m_node_output				= new float*[m_layer_num];
	m_node_output_prev			= new float*[m_layer_num];
	m_error						= new float*[m_layer_num];
	m_error_prev				= new float*[m_layer_num];

	for (int i = 0; i < m_layer_num; i++){
		*(m_node_output+i) = new float[m_neuron_num[i]];
		*(m_node_output_prev+i) = new float[m_neuron_num[i]];
		*(m_error+i) = new float[m_neuron_num[i]];
		*(m_error_prev+i) = new float[m_neuron_num[i]];
	}

	// --- END ---
	//////////////

	// Initialize m_error_prev and m_node_output variables
	for(int i=0;i<m_layer_num;i++)
		for(int j=0;j<m_neuron_num[i];j++){
			m_node_output[i][j] = 0.0F;
			m_error[i][j] = 0.0F;
			m_node_output_prev[i][j] = 0.0F;
			m_error_prev[i][j] = 0.0F;
		}

	// Initialize weights
	initialize_weights();
}

//SET

void CNeuralNetwork::ann_set_learning_rate(float learning_rate)
{
	m_learning_rate = learning_rate;
}

void CNeuralNetwork::ann_set_momentum(float momentum)
{
	m_momentum = momentum;
}

void CNeuralNetwork::ann_set_weight_init_method(int method, float range)
{
	m_method = method;
	m_init_val = range;
}

void CNeuralNetwork::ann_set_slope_value(float slope_value)
{
    m_slope = slope_value;
}

void CNeuralNetwork::ann_set_lr_changing_factor(float lr_factor)
{
    m_lr_factor = lr_factor;
}

void CNeuralNetwork::ann_set_input_per_channel(unsigned int input_channel, float input)
{
	m_current_input[input_channel] = input;
}

// GET

float CNeuralNetwork::ann_get_average_error()
{
    return m_average_error;
}

float CNeuralNetwork::ann_get_output(unsigned int channel)
{
	return m_node_output[m_layer_num-1][channel];
}

int CNeuralNetwork::ann_get_epoch_num()
{
    return m_epoch;
}

// TRAIN, TEST, SIMULATE AND CLEAR

void CNeuralNetwork::ann_train_network_from_file(char *file_name, int max_epoch, float max_error, int parsing_direction)
{
    string line;
	ifstream file (file_name);
	m_average_error = 0.0F;

	if (file.is_open()){
		for (m_epoch = 0; m_epoch < max_epoch; m_epoch++){
			int training_data_num = 0;
			float error = 0.0F;
			while (! file.eof() ){
				getline(file, line);
				if (line.empty())
					break;
				parse_data(line, parsing_direction);

				calculate_outputs();
				calculate_errors();
				calculate_weights();

				error = error + get_mse_error();
				training_data_num ++;
			}
			file.clear(); // clear buffer
			file.seekg(0, ios::beg); // go to begining of file

			float error_prev = m_average_error;
			m_average_error = error/training_data_num; // average of mse in one epoch

			if (m_average_error <= max_error)
				break;

			m_learning_rate = m_learning_rate*(m_lr_factor*m_average_error*error_prev + 1);
		}
	}
	file.close();
	generate_report();
}

void CNeuralNetwork::ann_test_network_from_file(char *file_name, char *log_file, int parsing_direction)
{
	string line;
	ifstream file (file_name);
	ofstream log (log_file);

	if (file.is_open()){
		while (! file.eof() ){
			getline(file, line);
			parse_data(line, parsing_direction);
			if (line.empty())
                break;

			calculate_outputs();
			calculate_errors();

			log << "expected output: ";
			for(int i = 0; i< m_neuron_num[m_layer_num-1]; i++){
                log << m_desired_output[i] << " ";
			}
            log << " calculated output: ";
            for(int i = 0; i< m_neuron_num[m_layer_num-1]; i++){
                log << ann_get_output(i) << " ";
			}
			log << " mse error : " <<  get_mse_error() << endl;
		}
	}
	file.close();
	log.close();
}

void CNeuralNetwork::ann_simulate()
{
	calculate_outputs();
}

void CNeuralNetwork::ann_clear()
{
	for (int i = 1; i < m_layer_num; i++){
		for (int j = 0; j < m_neuron_num[i]; j++)
			delete [] m_weight[i][j];
		delete [] m_weight[i];
	}
	delete [] m_weight;

	for (int i = 0; i < m_layer_num; i++){
		delete [] m_node_output[i];
		delete [] m_node_output_prev[i];
		delete [] m_error[i];
		delete [] m_error_prev[i];
	}
	delete [] m_node_output;
	delete [] m_node_output_prev;
	delete [] m_error;
	delete [] m_error_prev;

	delete [] m_current_input;
	delete [] m_desired_output;
	delete []m_neuron_num;

}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

auralius manurung
Gyeongsang National University, South Korea
Indonesia Indonesia
from Indonesia with love... Smile | :)

| Advertise | Privacy | Terms of Use | Mobile
Web03 | 2.8.150129.1 | Last Updated 12 Aug 2009
Article Copyright 2009 by auralius manurung
Everything else Copyright © CodeProject, 1999-2015
Layout: fixed | fluid