Multiple convolution neural networks approach for online handwriting recognition

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Drawing;
using System.Data;
using System.IO;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using Microsoft.Win32;
using System.Diagnostics;
using System.Security;
using System.Runtime.Serialization;
using DrawTools;
using DocToolkit;
using ANN.Perceptron;
using UPImage;
using System.Drawing.Imaging;
using ANN.Perceptron.ArchiveSerialization;
using ANN.Perceptron.Network;
using System.Threading;
using System.Threading.Tasks;
namespace NNControl.NNTesting
{
    public partial class NNTestingControl : Common.UPTemplateControl
    {
        #region Members
        private Color _color = Color.Black;
        private DocManager docManager;
        private DragDropManager dragDropManager;
        const string registryPath = "Software\\AlexF\\DrawTools";
        private Bitmap bitmap;
        private Bitmap drawBitmap;
        private List<ConvolutionNetwork> networks;
        #endregion

        #region Contruction
        public NNTestingControl()
        {
            InitializeComponent();
            bitmap = null;
            networks = null;
            textSpellControl1.SpellChecker = this.multipleSpelling;
            //English dicionary
            multipleSpelling.Dictionaries.Add(this.wordDictionary1);
            //France dictionary
            //multipleSpelling.Dictionaries.Add(this.wordDictionary2);
            //Italian dictionary
            //multipleSpelling.Dictionaries.Add(this.wordDictionary3);
           
        }
        private void NNTestingControl_Load(object sender, EventArgs e)
        {
            // Helper objects (DocManager and others)
            InitializeHelperObjects();
            LoadSettingsFromRegistry();
            // Initialize draw area
            //ResizeDrawArea();
            drawArea.Initialize(docManager);
            _color = DrawObject.LastUsedColor;
            btColor.BackColor = _color;
            udPenWidth.Value = DrawObject.LastUsedPenWidth;

        }
        #endregion

        #region DocManager Event Handlers

        /// <summary>
        /// DocManager reports hat it executed New command.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void docManager_ClearEvent(object sender, EventArgs e)
        {
            if (drawArea.GraphicsList != null)
            {
                drawArea.GraphicsList.Clear();
                drawArea.Refresh();
            }
        }

        /// <summary>
        /// DocManager reports that document was changed (loaded from file)
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void docManager_DocChangedEvent(object sender, EventArgs e)
        {
            drawArea.Refresh();
        }

        /// <summary>
        /// DocManager reports about successful/unsuccessful Open File operation
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void docManager_OpenEvent(object sender, OpenFileEventArgs e)
        {
  
        }

        /// <summary>
        /// Load document from the stream supplied by DocManager
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="args"></param>
        private void docManager_LoadEvent(object sender, SerializationEventArgs e)
        {
            // DocManager asks to load document from supplied stream
            try
            {
                drawArea.GraphicsList = (GraphicsList)e.Formatter.Deserialize(e.SerializationStream);
            }
            catch (ArgumentNullException ex)
            {
                HandleLoadException(ex, e);
            }
            catch (SerializationException ex)
            {
                HandleLoadException(ex, e);
            }
            catch (SecurityException ex)
            {
                HandleLoadException(ex, e);
            }
        }


        /// <summary>
        /// Save document to stream supplied by DocManager
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="args"></param>
        private void docManager_SaveEvent(object sender, SerializationEventArgs e)
        {
            // DocManager asks to save document to supplied stream
            try
            {
                e.Formatter.Serialize(e.SerializationStream, drawArea.GraphicsList);
            }
            catch (ArgumentNullException ex)
            {
                HandleSaveException(ex, e);
            }
            catch (SerializationException ex)
            {
                HandleSaveException(ex, e);
            }
            catch (SecurityException ex)
            {
                HandleSaveException(ex, e);
            }
        }

        #endregion

        #region DragDropManager Event Handlers

        private void dragDropManager_FileDroppedEvent(object sender, FileDroppedEventArgs e)
        {
            OpenDocument(e.FileArray.GetValue(0).ToString());
        }

        #endregion

        #region Other Functions

        /// <summary>
        /// Set Pointer draw tool
        /// </summary>
        private void CommandPointer()
        {
            drawArea.ActiveTool = DrawToolType.Pointer;
        }

        /// <summary>
        /// Set Rectangle draw tool
        /// </summary>
        private void CommandRectangle()
        {
            drawArea.ActiveTool = DrawToolType.Rectangle;
        }

        /// <summary>
        /// Set Ellipse draw tool
        /// </summary>
        private void CommandEllipse()
        {
            drawArea.ActiveTool = DrawToolType.Ellipse;
        }

        /// <summary>
        /// Set Line draw tool
        /// </summary>
        private void CommandLine()
        {
            drawArea.ActiveTool = DrawToolType.Line;
        }

        /// <summary>
        /// Set Polygon draw tool
        /// </summary>
        private void CommandPolygon()
        {
            drawArea.ActiveTool = DrawToolType.Polygon;
        }

        /// <summary>
        /// Show About dialog
        /// </summary>
        private void CommandAbout()
        {
          
        }

        /// <summary>
        /// Set draw area to all form client space except toolbar
        /// </summary>
        private void ResizeDrawArea()
        {
            Rectangle rect = this.ClientRectangle;

            drawArea.Left = rect.Left;
            drawArea.Top = rect.Top ;
            drawArea.Width = rect.Width;
            drawArea.Height = rect.Height;
        }

        /// <summary>
        /// Load application settings from the Registry
        /// </summary>
        void LoadSettingsFromRegistry()
        {
            try
            {
                RegistryKey key = Registry.CurrentUser.CreateSubKey(registryPath);

                DrawObject.LastUsedColor = Color.FromArgb((int)key.GetValue(
                    "Color",
                    Color.Black.ToArgb()));

                DrawObject.LastUsedPenWidth = (int)key.GetValue(
                    "Width",
                    1);
                drawArea.ActiveTool = (DrawToolType)key.GetValue("Activetool", DrawToolType.Pointer);
            }
            catch (ArgumentNullException ex)
            {
                HandleRegistryException(ex);
            }
            catch (SecurityException ex)
            {
                HandleRegistryException(ex);
            }
            catch (ArgumentException ex)
            {
                HandleRegistryException(ex);
            }
            catch (ObjectDisposedException ex)
            {
                HandleRegistryException(ex);
            }
            catch (UnauthorizedAccessException ex)
            {
                HandleRegistryException(ex);
            }
        }

        /// <summary>
        /// Save application settings to the Registry
        /// </summary>
        public void SaveSettingsToRegistry()
        {
            try
            {
                RegistryKey key = Registry.CurrentUser.CreateSubKey(registryPath);

                key.SetValue("Color", DrawObject.LastUsedColor.ToArgb());
                key.SetValue("Width", DrawObject.LastUsedPenWidth);
                key.SetValue("Activetool",(int) drawArea.ActiveTool);
            }
            catch (SecurityException ex)
            {
                HandleRegistryException(ex);
            }
            catch (ArgumentException ex)
            {
                HandleRegistryException(ex);
            }
            catch (ObjectDisposedException ex)
            {
                HandleRegistryException(ex);
            }
            catch (UnauthorizedAccessException ex)
            {
                HandleRegistryException(ex);
            }
        }

        private void HandleRegistryException(Exception ex)
        {
            Trace.WriteLine("Registry operation failed: " + ex.Message);
        }

        /// <summary>
        /// Open new file
        /// </summary>
        private void CommandNew()
        {
            docManager.NewDocument();
        }

        /// <summary>
        /// Open file
        /// </summary>
        private void CommandOpen()
        {
            docManager.OpenDocument("");
        }

        /// <summary>
        /// Save file
        /// </summary>
        private void CommandSave()
        {
            docManager.SaveDocument(DocManager.SaveType.Save);
        }

        /// <summary>
        /// Save As
        /// </summary>
        private void CommandSaveAs()
        {
            docManager.SaveDocument(DocManager.SaveType.SaveAs);
        }

        /// <summary>
        /// Handle exception from docManager_LoadEvent function
        /// </summary>
        /// <param name="ex"></param>
        /// <param name="fileName"></param>
        private void HandleLoadException(Exception ex, SerializationEventArgs e)
        {
            MessageBox.Show(this,
                "Open File operation failed. File name: " + e.FileName + "\n" +
                "Reason: " + ex.Message,
                Application.ProductName);

            e.Error = true;
        }

        /// <summary>
        /// Handle exception from docManager_SaveEvent function
        /// </summary>
        /// <param name="ex"></param>
        /// <param name="fileName"></param>
        private void HandleSaveException(Exception ex, SerializationEventArgs e)
        {
            MessageBox.Show(this,
                "Save File operation failed. File name: " + e.FileName + "\n" +
                "Reason: " + ex.Message,
                Application.ProductName);

            e.Error = true;
        }

        /// <summary>
        /// Initialize helper objects from the DocToolkit Library.
        /// 
        /// Called from Form1_Load. Initialized all objects except
        /// PersistWindowState wich must be initialized in the
        /// form constructor.
        /// </summary>
        private void InitializeHelperObjects()
        {
            // DocManager

            DocManagerData data = new DocManagerData();
            data.FormOwner = this.ParentForm;
            data.UpdateTitle = true;
            data.FileDialogFilter = "DrawTools files (*.dtl)|*.dtl|All Files (*.*)|*.*";
            data.NewDocName = "Untitled.dtl";
            data.RegistryPath = registryPath;

            docManager = new DocManager(data);
            docManager.RegisterFileType("dtl", "dtlfile", "DrawTools File");

            docManager.SaveEvent += new SaveEventHandler(docManager_SaveEvent);
            docManager.LoadEvent += new LoadEventHandler(docManager_LoadEvent);
            docManager.OpenEvent += new OpenFileEventHandler(docManager_OpenEvent);
            docManager.DocChangedEvent += new EventHandler(docManager_DocChangedEvent);
            docManager.ClearEvent += new EventHandler(docManager_ClearEvent);

            docManager.NewDocument();

            // DragDropManager
            dragDropManager = new DragDropManager(this.ParentForm);
            dragDropManager.FileDroppedEvent += new FileDroppedEventHandler(this.dragDropManager_FileDroppedEvent);

        }

        /// <summary>
        /// Open document.
        /// Used to open file passed in command line or dropped into the window
        /// </summary>
        /// <param name="file"></param>
        public void OpenDocument(string file)
        {
            docManager.OpenDocument(file);
        }

        #endregion

        #region Event Handlers
        private void btNewFile_Click(object sender, EventArgs e)
        {
            CommandNew();


        }

        private void btnOpen_Click(object sender, EventArgs e)
        {
            CommandOpen();
            
        }

        private void btSave_Click(object sender, EventArgs e)
        {
            CommandSave();
        }

        private void btLine_Click(object sender, EventArgs e)
        {
            CommandLine();
        }

        private void btPencil_Click(object sender, EventArgs e)
        {
            CommandPolygon(); 
        }

        private void btRectangle_Click(object sender, EventArgs e)
        {
            CommandRectangle();
        }

        private void btEllipse_Click(object sender, EventArgs e)
        {
            CommandEllipse();
        }

        private void btPointer_Click(object sender, EventArgs e)
        {
            CommandPointer();
        }

        private void btColor_Click(object sender, EventArgs e)
        {
            ColorDialog dlg = new ColorDialog();
            dlg.Color = _color;

            if (dlg.ShowDialog(this) == DialogResult.OK)
            {
                btColor.BackColor = dlg.Color;
                _color = dlg.Color;
                drawArea.GraphicsList.RedrawSelectedDrawObjects(_color);
                drawArea.Refresh();
            }
        }

        private void udPenWidth_ValueChanged(object sender, EventArgs e)
        {
            drawArea.GraphicsList.RedrawSelectedDrawObjects((int)udPenWidth.Value);
            drawArea.Refresh();
        }
        private void btExport_Click(object sender, EventArgs e)
        {
            Bitmap bmp = new Bitmap(drawArea.Width, drawArea.Height);
            drawArea.DrawToBitmap(bmp, new Rectangle(0, 0, bmp.Width, bmp.Height));
            SaveFileDialog saveDlg = new SaveFileDialog();
            saveDlg.Filter = "Bitmap files (*.bmp)|*.bmp";
            saveDlg.RestoreDirectory = true;
            bmp = Skeletonization.MedialAxisTransform(bmp);
            if (saveDlg.ShowDialog() == DialogResult.OK)
            {
                bmp.Save(saveDlg.FileName);
            }
            
        }
     
        private void btRecognition_Click(object sender, EventArgs e)
        {
            if (networks != null)
            {
                //recognition all characters in the drawArea
                if (bitmap != null)
                {
                    bitmap.Dispose();
                    bitmap = null;
                }
                if (drawBitmap != null)
                {
                    drawBitmap.Dispose();
                    drawBitmap = null;
                }
                bitmap = new Bitmap(drawArea.Width, drawArea.Height);
                drawArea.DrawToBitmap(bitmap, new Rectangle(0, 0, bitmap.Width, bitmap.Height));
                drawBitmap = (Bitmap)bitmap.Clone();
                if (bitmap != null)
                {
                    List<InputPattern> lineList = null;
                    InputPattern parentPt = new InputPattern(bitmap, 255, new Rectangle(0, 0, bitmap.Width, bitmap.Height));
                    lineList = GetPatternsFromBitmap(parentPt, 500, 1, true, 10, 10);
                    if (lineList != null)
                    {
                        if (lineList.Count > 0)
                        {

                            String text = "";
                            foreach (var line in lineList)
                            {
                                List<InputPattern> wordList = null;
                                wordList = GetPatternsFromBitmap(line, 100, 10, false, 10, 10);
                                String lineText = "";
                                if (wordList != null)
                                {
                                    if (wordList.Count > 0)
                                    {
                                        foreach (var word in wordList)
                                        {
                                            WordRecognition wr = new WordRecognition(multipleSpelling);
                                            wr.Networks = networks;
                                            if (lineText == "")
                                            {
                                                lineText = wr.GetBestRecognizedWord(word).Trim();
                                            }
                                            else
                                            {
                                                lineText = String.Format("{0} {1}", lineText, wr.GetBestRecognizedWord(word).Trim());
                                            }
                                            //Draw word rectangle
                                            //drawBitmap = word.DrawPatternBoundary(drawBitmap, Color.Red);
                                            drawBitmap = word.DrawChildPatternBoundaries(drawBitmap);
                                            wr = null;
                                        }
                                        wordList.Clear();
                                        wordList = null;
                                    }

                                }
                                lineText = String.Format("{0}{1}", lineText, "\n");
                                text = String.Format("{0}{1}", text, lineText);
                            }
                            textSpellControl1.SpellCheckingText = text;
                            pbPreview.Image = drawBitmap;
                            //Bitmap temp= ImageProcessing.Contuor(bitmap);
                            //temp = bitmap;
                            //double k = Projections.GetMinimumThickofHorizonHistogram(drawBitmap, 50);
                            //temp=ImageProcessing.MarkIndexedPattern(temp,(int) k, 255);
                            //pbPreview.Image = temp;
                            lineList.Clear();
                            lineList = null;
                            
                        }

                    }
                    parentPt.Dispose();
                    parentPt = null;
                }
            }
        }
        private bool PatternRecognition(Bitmap orginal,out Char accChar)
        {
            bool isCharFound = false;
            accChar = new Char();
            if (networks != null)
            {
                double ratio;
                int w;
                int h;
                Bitmap bm;
                int widthpad = 0;
                int heightpad = 0;
                if (orginal.Width > orginal.Height)
                {
                    ratio = (double)orginal.Width / 25;
                    w = 25;
                    h = (int)Math.Floor(orginal.Height / ratio);
                    bm = UPImage.ImageProcessing.ResizeBitmap(orginal, w, h);
                    widthpad = 4;
                    heightpad = 29 - h;
                }
                else
                {
                    ratio = (double)orginal.Height / 25;
                    h = 25;
                    w = (int)Math.Floor(orginal.Width / ratio);
                    bm = UPImage.ImageProcessing.ResizeBitmap(orginal, w, h);
                    widthpad = 29 - w;
                    heightpad = 4;
                }
                bm = UPImage.ImageProcessing.CreateColorPad(bm, Color.White, widthpad, heightpad);
                bm = UPImage.ImageProcessing.ColorToGrayscale(bm);
                byte[] imagedata = ImageProcessing.ConvertGrayscaleBitmaptoBytes(bm);


                foreach (var nw in networks)
                {
                    PatternRecognition patterntest = new PatternRecognition(nw, this);
                    patterntest.ForwardpropagationThread(imagedata, bm.Width, bm.Height, out accChar);
                    if (accChar != nw.UnknownOuput)
                    {
                        isCharFound = true;
                        break;
                    }
                }
                
            }
            return isCharFound;
        }
        private void btLoad_Click(object sender, EventArgs e)
        {
            toolStripStatusLabel.Text = "loading network files...";
            if (LoadTrainedParametersFiles())
            {
                toolStripStatusLabel.Text = "Networks loaded successfully!";
            }
            else
            {
                toolStripStatusLabel.Text = "Networks loaded fail!";
            }
        }
        private void btnNext_Click(object sender, EventArgs e)
        {
            
        }
        private void btnBack_Click(object sender, EventArgs e)
        {
           
        }
        #endregion

        #region Functions

        Bitmap DrawTexttoBitmap(Bitmap orginal, String text)
        {
            Bitmap bmp = (Bitmap)orginal.Clone();
            Graphics graph = Graphics.FromImage(bmp);
            Font font = new Font(Font.FontFamily, 30, FontStyle.Bold);
            TextRenderer.DrawText(graph, text, font, new Point(1, 1), Color.Black);
            graph.DrawImage(bmp, Point.Empty);
            return bmp;
        }
        private List<InputPattern> GetPatternsFromBitmap(InputPattern parentPattern, int hStep, int vStep, bool isTopStart, int minWidth, int minHeight)
        {
            List<InputPattern> patternlist = null;
            parentPattern.GetPatternBoundaries(hStep, vStep, isTopStart, minWidth, minHeight);
            if (parentPattern.PatternRects.Count > 0)
            {
                patternlist = new List<InputPattern>();
                foreach (var rect in parentPattern.PatternRects)
                {
                    Bitmap bmp = new Bitmap(rect.Width, rect.Height);
                    Graphics graph = Graphics.FromImage(bmp);
                    graph.DrawImage(parentPattern.OriginalBmp, 0, 0, rect, GraphicsUnit.Pixel);
                    int x=parentPattern.OriginalRectangle.X+rect.X;
                    int y=parentPattern.OriginalRectangle.Y+rect.Y;
                    Rectangle newRect = new Rectangle(x, y, rect.Width, rect.Height);
                    InputPattern childPattern = new InputPattern(bmp, 255, newRect);
                    patternlist.Add(childPattern);
                    graph.Dispose();
                }
          
            }
            
            return patternlist;
        }
        /// <summary>
        /// Load trained parameters files to neural networks
        /// </summary>
        /// <returns></returns>
        private bool LoadTrainedParametersFiles()
        {
            OpenFileDialog OpenFileDialog1 = new System.Windows.Forms.OpenFileDialog { Filter = "Neural network parameters file (*.nnt)|*.nnt", Title = "Load Neural network File" };
            OpenFileDialog1.Multiselect = true;
            bool result = false;
            try
            {
                if (OpenFileDialog1.ShowDialog() == DialogResult.OK)
                {
                    // Read the files
                    if (networks != null)
                    {
                        networks.Clear();
                        networks = null;
                    }
                    Task task = Task.Factory.StartNew(() =>
                    {
                        networks = new List<ConvolutionNetwork>();
                        foreach (String file in OpenFileDialog1.FileNames)
                        {
                            // Create a PictureBox.
                            try
                            {
                                ConvolutionNetwork nw = new ConvolutionNetwork();
                                var fsIn = new StreamReader(file);
                                var arIn = new Archive(fsIn.BaseStream, ArchiveOp.load);
                                nw.Serialize(arIn);
                                fsIn.Close();
                                networks.Add(nw);
                            }
                            catch (Exception ex)
                            {

                                result = false;
                            }
                        }
                    });
                    task.Wait();
                    result = true;
                }
            }
            finally
            {
                if (OpenFileDialog1 != null)
                    OpenFileDialog1.Dispose();
            }
            return result;
        }

        private void btnPreview_Click(object sender, EventArgs e)
        {
            spcMain.Panel2Collapsed = !spcMain.Panel2Collapsed;
        }
        /// <summary>
        /// recognize a bitmap 
        /// </summary>
        /// <param name="orginal"></param>
        #endregion
    }
}