Back-propagation Algorithm for 3 layers - multiple input and multiple output code

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace BackProbagationAlgor
{
    class Program
    {
        // step 1: initilize data
        static double w = 0.02; // connection weight
        static double Emax = 0.0001; // max error tolerate
        static double E = 0; // initial error
        static double zeta = 0.02; // learning rate

        static int numberInputNeuron = 3;
        // static int numberHiddenLayer = 1;
        static int numberHiddenNodeInEachLayer = 6;
        static int numberOutputNeuron = 2;
        static int numberTrainingPatterns = 4;

        static double[] errThisPat = new double[numberOutputNeuron];// set to 1 as first start
        static int patNum = 0;

        static double[] deltaO = new double[numberOutputNeuron]; // error signal for output layer
        static double[] deltaH = new double[numberHiddenNodeInEachLayer];
        static int epoch = 0; // count number of epoch. Has to go to 1 epoch at lease

        static void Main(string[] args)
        {
            try
            {
                epoch = 0;
                initData();

                //while (errThisPat[1] >= Emax || errThisPat[2] >= Emax || epoch < 1)
                while (errThisPat[0] >= Emax || errThisPat[1] > Emax || epoch < 1)
                {
                    for (int i = 0; i < numberTrainingPatterns; i++)
                    {
                        patNum = i;
                        trainData();
                    }
                    epoch++;
                }

                // Write to display the connection weight after all calculation
                for (int i = 0; i < numberOutputNeuron; i++)
                {
                    for (int j = 0; j < numberHiddenNodeInEachLayer; j++)
                    {
                        Console.WriteLine("Weight" + i + j + " is " + weightHO[i, j]);
                    }
                }

                for (int k = 0; k < numberHiddenNodeInEachLayer; k++)
                {
                    for (int l = 0; l < numberInputNeuron; l++)
                    {
                        Console.WriteLine("Weight" + k + l + " is " + weightIH[k, l]);
                    }
                }
                // end write display connection weight
            }
            catch
            {
                Console.WriteLine("Eror at Main");
            }

            tryOutPut();

            Console.Read();
        }

        static double[,] input = new double[numberTrainingPatterns, numberInputNeuron]; // input data array
        static double[,] output = new double[numberTrainingPatterns, numberOutputNeuron]; // output data array

        static double[,] try_input = new double[4, numberInputNeuron]; // set of test data input to see if the network can produce the correct result after trained
        static double[,] try_output = new double[4, numberOutputNeuron]; // // set of test data input to see if the network can produce the correct result after trained

        static double[] outputOfEachNuron = new double[numberHiddenNodeInEachLayer]; // place holder for calculating of output at each neuron in the hidden layer
        static double[,] weightIH = new double[numberHiddenNodeInEachLayer, numberInputNeuron]; // connective weight between input and hidden nodes
        static double[,] weightHO = new double[numberOutputNeuron, numberHiddenNodeInEachLayer]; // connective weight between hidden and output nodes
        static double[,] outputTemp = new double[numberHiddenNodeInEachLayer, numberOutputNeuron];// temporay array to store the output of each hidden node

        static Random randNum = new Random();
        // step 2: apply input
        public static void initData()
        {
            // set learning rate for HO
            for (int i = 0; i < numberOutputNeuron; i++)
            {
                errThisPat[i] = 1;//initilize error compued from a pattern of data set
                for (int j = 0; j < numberHiddenNodeInEachLayer; j++)
                {
                    weightHO[i, j] = randNum.NextDouble();
                }
            }

            // set learning rate for IH
            for (int k = 0; k < numberHiddenNodeInEachLayer; k++)
            {
                for (int l = 0; l < numberInputNeuron; l++)
                {
                    weightIH[k, l] = randNum.NextDouble();
                }
            }

            // training data
            input[0, 0] = 1;
            input[0, 1] = -1;
            input[0, 2] = 1;
            output[0, 0] = 1; // xor
            output[0, 1] = 1; // xor

            input[1, 0] = -1;
            input[1, 1] = 1;
            input[1, 2] = 1;
            output[1, 0] = 1;
            output[1, 1] = 1;

            input[2, 0] = 1;
            input[2, 1] = 1;
            input[2, 2] = 1;
            output[2, 0] = -1;
            output[2, 1] = -1;

            input[3, 0] = -1;
            input[3, 1] = -1;
            input[3, 2] = 1;
            output[3, 0] = -1;
            output[3, 1] = -1;
        }

        static int flag = 0;
        // static double[] delta
        public static void trainData()
        {
            try
            {
                // step 3: forward probagation
                // calculate the output of each neuron in the hidden layer
                for (int i = 0; i < numberHiddenNodeInEachLayer; i++)
                {
                    outputOfEachNuron[i] = 0; // clare array before use
                    for (int j = 0; j < numberInputNeuron; j++)
                    {
                        outputOfEachNuron[i] = outputOfEachNuron[i] + (input[patNum, j] * weightIH[i, j]);
                    }
                    outputOfEachNuron[i] = Math.Tanh(outputOfEachNuron[i]);
                }

                // calculate the output of each neuron in the output layer
                for (int i = 0; i < numberOutputNeuron; i++)
                {
                    flag = i;
                    //outputTemp[i, 0] = 0;
                    double outputTempHolder = 0;
                    for (int j = 0; j < numberHiddenNodeInEachLayer; j++)
                    {
                        outputTempHolder = outputTempHolder + outputOfEachNuron[j] * weightHO[i, j]; // output of 1 output neuron
                    }

                    // step 4: calculate the error
                    errThisPat[i] = 0.5 * Math.Pow(output[patNum, i] - outputTempHolder, 2) + E;
                    Console.WriteLine("Epoch = " + epoch + " EMS" + i + " = " + errThisPat[i]);

                    // compute the error signal of the output neuron
                    deltaO[i] = (1 + outputTempHolder) * (1 - outputTempHolder);// derivative of tanh

                    // step 5: Back probagation
                    // update weight of 3rd layer
                    updateWeightHO(deltaO[i], i);
                }

                // 2nd layer update
                updateWeightIH();
            }
            catch
            {
                Console.WriteLine("error");
            }
        }

        public static void updateWeightHO(double _delta, int n)
        {
            try
            {
                // 3rd layer weigh update
                for (int j = 0; j < numberHiddenNodeInEachLayer; j++)
                {
                    double weightChangeHO = zeta * _delta * outputOfEachNuron[j];
                    weightHO[n, j] = weightChangeHO + weightHO[n, j];
                }
            }
            catch { Console.WriteLine("Eror updateWeightHO"); }
        }

        public static void updateWeightIH()
        {
            try
            {
                for (int i = 0; i < numberHiddenNodeInEachLayer; i++)
                {
                    double temp = (1 + outputOfEachNuron[i]) * (1 - outputOfEachNuron[i]);//outputOfEachNuron[i] * (1 - outputOfEachNuron[i]);// holds the temp output of the tanh-derivative which is [(1-O)*(1+O)]
                    deltaH[i] = 0; // not sufficient - may need to clear the array at the beginnin of this loop
                    for (int j = 0; j < numberOutputNeuron; j++)
                    {
                        deltaH[i] += weightHO[j, i] * deltaO[j];
                    }
                    deltaH[i] = deltaH[i] * temp;
                }

                // update 2nd layer weight
                for (int k = 0; k < numberInputNeuron; k++)
                {
                    for (int l = 0; l < numberHiddenNodeInEachLayer; l++)
                    {
                        weightIH[l, k] = (zeta * deltaH[l] * input[patNum, k]) + weightIH[l, k]; // weight_new = weight_old + delta_weight
                    }
                }
            }
            catch { Console.WriteLine("Eror updateWeightIH"); }
        }

        static double[,] tryouput = new double[4, numberOutputNeuron];// should change 4 to correct number of data of try_output
        
        // this method will only run after the error converge, so for now, not to worry about this method
        public static void tryOutPut()
        {
            // testing data to see if ANN gives correct result after trained
            try_input[0, 0] = 1;
            try_input[0, 1] = -1;
            try_input[0, 2] = 1;
            try_output[0, 0] = 1; // xor
            try_output[0, 1] = 1; // xor

            try_input[1, 0] = -1;
            try_input[1, 1] = 1;
            try_input[1, 2] = 1;
            try_output[1, 0] = 1;
            try_output[1, 1] = 1;

            try_input[2, 0] = 1;
            try_input[2, 1] = 1;
            try_input[2, 2] = 1;
            try_output[2, 0] = -1;
            try_output[2, 1] = -1;

            try_input[3, 0] = -1;
            try_input[3, 1] = -1;
            try_input[3, 2] = 1;
            try_output[3, 0] = -1;
            try_output[3, 1] = -1;

            int tryoutputPat = 4;

            for (int x = 0; x < tryoutputPat; x++) // loop through all the try_ouput data sets
            {
                // calculate the output of each neuron in the hidden layer
                for (int i = 0; i < numberHiddenNodeInEachLayer; i++)
                {
                    outputOfEachNuron[i] = 0; // clare array before use
                    for (int j = 0; j < numberInputNeuron; j++)
                    {
                        outputOfEachNuron[i] = outputOfEachNuron[i] + (input[x, j] * weightIH[i, j]);
                    }
                    outputOfEachNuron[i] = Math.Tanh(outputOfEachNuron[i]);
                }

                // calculate the output of each neuron in the output layer
                for (int i = 0; i < numberOutputNeuron; i++)
                {
                    flag = i;
                    //outputTemp[i, 0] = 0;
                    double outputTempHolder = 0;
                    for (int j = 0; j < numberHiddenNodeInEachLayer; j++)
                    {
                        outputTempHolder = outputTempHolder + outputOfEachNuron[j] * weightHO[i, j]; // output of 1 output neuron
                        Console.WriteLine("");
                    }
                    tryouput[x, i] = outputTempHolder;
                    Console.WriteLine(" Set " + x + ": Target " + i + " is: " + try_output[x, i] + " ANN is " + tryouput[x, i]);
                }
            }
        }
    }
}