Click here to Skip to main content
15,884,628 members
Articles / Artificial Intelligence / Neural Networks

Multiple convolution neural networks approach for online handwriting recognition

Rate me:
Please Sign up or sign in to vote.
4.95/5 (37 votes)
9 Apr 2013CPOL8 min read 75.9K   25.1K   74  
The research focuses on the presentation of word recognition technique for an online handwriting recognition system which uses multiple component neural networks (MCNN) as the exchangeable parts of the classifier.
// UNIPEN.NET framework
//
// Copyright © Pham Viet Dung, 2011-2012
// vietdungiitb@vimaru.edu.vn
//

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Windows;
using System.Drawing;
using System.Drawing.Imaging;
using UPImage.FileFormat;
namespace UPImage
{
    public enum DrawType
    {
        Line,
        Pie
    }

    public class UnipenBitmap
    {
        #region private properties
        // Define parameters used to create the BitmapSource.
        PixelFormat pf;
        int width;
        int height;
        int penwidth;
        bool bmulticolor;
        bool uppen;
        bool isBackgroundColor;
        DrawType drawtype;
        UPUnipen unipien;
        List<Color> colorList;

        #endregion

        #region public properties

        public bool IsBackgroundColor
        {
            get
            {
                return isBackgroundColor;
            }
            set
            {
                if (isBackgroundColor == value)
                    return;
                isBackgroundColor = value;
            }
        }
        public Bitmap Image
        {
            get
            {
                if (unipien.PenTrajectories.Count > 0)
                {

                    return GetViewedBitmap();
                    //return GetGrayscaleBitmap(6, new Size(61, 61));
                }
                else
                {
                    return null;
                }
            }
        }
        public int Width
        {
            get
            {
                return width;
            }
        }
        public int Height
        {
            get
            {
                return height;
            }
        }
        public PixelFormat Format
        {
            get
            {
                return pf;
            }
            set
            {
                pf = (PixelFormat)value;
            }
        }
        //
        public int PenWidth
        {
            get
            {
                return penwidth;
            }
            set
            {
                if (penwidth == value)
                    return;
                penwidth = value;
            }
        }
        public bool Multicolor
        {
            get
            {
                return bmulticolor;
            }
            set
            {
                if (bmulticolor == value)
                    return;
                bmulticolor = value;
            }
        }
        public DrawType Drawtype
        {
            get
            {
                return drawtype;
            }
            set
            {
                if (drawtype == value)
                    return;
                drawtype = value;
            }
        }
        public bool Uppen
        {
            get
            {
                return uppen;
            }
            set
            {
                if (uppen == value)
                    return;
                uppen = value;
            }
        }

        #endregion

        #region constructors
        /// <summary>
        ///
        /// </summary>
        /// <param name="up"> UPUnipen object</param>
        /// <param name="pw"> width of pen to draw UPUnipen object</param>
        public UnipenBitmap(UPUnipen up, int pw)
        {
            InitColorList();
            penwidth = pw;
            unipien = up;
            bmulticolor = true;
            drawtype = DrawType.Line;
            uppen = false;
            isBackgroundColor = false;
        }
        public UnipenBitmap(UPUnipen up)
        {
            InitColorList();
            penwidth = 20;
            unipien = up;
            bmulticolor = true;
            drawtype = DrawType.Line;
            uppen = false;
            isBackgroundColor = false;
        }

        #endregion

        #region private methods

        private Bitmap GetViewedBitmap()
        {
            if (unipien.PenTrajectories.Count > 0)
            {
                Size size = new Size(20 + (unipien.Bottom.X - unipien.Top.X), 20 + (unipien.Bottom.Y - unipien.Top.Y));
                pf = PixelFormat.Format32bppArgb;
                Bitmap bmp = new Bitmap(size.Width, size.Height, pf);
                width = bmp.Width;
                height = bmp.Height;
                Graphics graphicGraph = Graphics.FromImage(bmp);
                SolidBrush brush = new SolidBrush(colorList[0]);
                Pen colorPen = new Pen(brush, 2);
                //draw bitmap border
                if (isBackgroundColor)
                {
                    SolidBrush backgroundBrush = new SolidBrush(Color.White);
                    graphicGraph.FillRectangle(backgroundBrush, new Rectangle(Point.Empty, bmp.Size));
                }
                int icolor = 0;
                foreach (var pt in unipien.PenTrajectories)
                {
                    if (pt.Keyname == UPImage.Common.UpCommonLib.keyPenDown)
                    {
                        brush = new SolidBrush(colorList[icolor]);
                        colorPen = new Pen(brush, penwidth);
                        if (pt.Points.Count > 0)
                        {
                            Point prepoint = pt.Points[0];
                            prepoint.X = prepoint.X - unipien.Top.X + 10;
                            prepoint.Y = (unipien.Bottom.Y - prepoint.Y) + 10;
                            foreach (var point in pt.Points)
                            {
                                int x = point.X - unipien.Top.X + 10;
                                int y = (unipien.Bottom.Y - point.Y) + 10;
                                Point p = new Point(x, y);
                                if (drawtype == DrawType.Line)
                                {
                                    graphicGraph.DrawLine(colorPen, prepoint, p);
                                    prepoint = p;
                                }
                                else
                                    if (drawtype == DrawType.Pie)
                                    {
                                        graphicGraph.FillPie(brush, x - penwidth / 2, y - penwidth / 2, penwidth, penwidth, 0, 360);
                                    }
                            }
                            if (bmulticolor)
                            {
                                if (icolor < colorList.Count - 1)
                                {
                                    icolor++;
                                }
                                else
                                {
                                    icolor = 0;
                                }
                            }
                        }
                    }
                    else
                    {
                        if (uppen)
                        {
                            brush = new SolidBrush(colorList[icolor]);
                            colorPen = new Pen(brush, penwidth);
                            if (pt.Points.Count > 0)
                            {
                                Point prepoint = pt.Points[0];
                                prepoint.X = prepoint.X - unipien.Top.X + 10;
                                prepoint.Y = (unipien.Bottom.Y - prepoint.Y) + 10;
                                foreach (var point in pt.Points)
                                {
                                    int x = point.X - unipien.Top.X + 10;
                                    int y = (unipien.Bottom.Y - point.Y) + 10;
                                    Point p = new Point(x, y);
                                    // present up pen by pies
                                    graphicGraph.FillPie(brush, x - penwidth / 2, y - penwidth / 2, penwidth, penwidth, 0, 360);

                                }
                                if (bmulticolor)
                                {
                                    if (icolor < colorList.Count - 1)
                                    {
                                        icolor++;
                                    }
                                    else
                                    {
                                        icolor = 0;
                                    }
                                }
                            }
                        }
                    }
                }
                return bmp;
            }
            else
            {
                return null;
            }
            
        }
        private void InitColorList()
        {
            colorList = new List<Color>();
            foreach (string colorName in Enum.GetNames(typeof(System.Drawing.KnownColor)))
            {
                //Check if color is dark 
                if (colorName.StartsWith("D") == true)
                {
                    colorList.Add(System.Drawing.Color.FromName(colorName));
                }
            }
        }
        private Bitmap MakeGrayscale(Bitmap original)
        {
            //create a blank bitmap the same size as original
            Bitmap newBitmap = new Bitmap(original.Width, original.Height);

            //get a graphics object from the new image
            Graphics g = Graphics.FromImage(newBitmap);

            //create the grayscale ColorMatrix
            ColorMatrix colorMatrix = new ColorMatrix(
               new float[][] 
              {
                 new float[] {.3f, .3f, .3f, 0, 0},
                 new float[] {.59f, .59f, .59f, 0, 0},
                 new float[] {.11f, .11f, .11f, 0, 0},
                 new float[] {0, 0, 0, 1, 0},
                 new float[] {0, 0, 0, 0, 1}
              });

            //create some image attributes
            ImageAttributes attributes = new ImageAttributes();

            //set the color matrix attribute
            attributes.SetColorMatrix(colorMatrix);

            //draw the original image on the new image
            //using the grayscale color matrix
            g.DrawImage(original, new Rectangle(0, 0, original.Width, original.Height),
               0, 0, original.Width, original.Height, GraphicsUnit.Pixel, attributes);

            //dispose the Graphics object
            g.Dispose();
            return newBitmap;
        }
        private  Bitmap ColorToGrayscale(Bitmap original)
        {
            int w = original.Width,
                h = original.Height,
                r, ic, oc, bmpStride, outputStride, bytesPerPixel;
            PixelFormat pfIn = original.PixelFormat;
            ColorPalette palette;
            Bitmap output;
            BitmapData bmpData, outputData;

            //Create the new bitmap
            output = new Bitmap(w, h, PixelFormat.Format8bppIndexed);

            //Build a grayscale color Palette
            palette = output.Palette;
            for (int i = 0; i < 256; i++)
            {
                Color tmp = Color.FromArgb(255, i, i, i);
                palette.Entries[i] = Color.FromArgb(255, i, i, i);
            }
            output.Palette = palette;

            //No need to convert formats if already in 8 bit
            if (pfIn == PixelFormat.Format8bppIndexed)
            {
                output = (Bitmap)original.Clone();

                //Make sure the palette is a grayscale palette and not some other
                //8-bit indexed palette
                output.Palette = palette;

                return output;
            }

            //Get the number of bytes per pixel
            switch (pfIn)
            {
                case PixelFormat.Format24bppRgb: bytesPerPixel = 3; break;
                case PixelFormat.Format32bppArgb: bytesPerPixel = 4; break;
                case PixelFormat.Format32bppRgb: bytesPerPixel = 4; break;
                default: throw new InvalidOperationException("Image format not supported");
            }

            //Lock the images
            bmpData = original.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.ReadOnly,
                                   pfIn);
            outputData = output.LockBits(new Rectangle(0, 0, w, h), ImageLockMode.WriteOnly,
                                         PixelFormat.Format8bppIndexed);
            bmpStride = bmpData.Stride;
            outputStride = outputData.Stride;
            int bmpBytes = Math.Abs(bmpData.Stride) * original.Height;
            byte[] rgbValues = new byte[bmpBytes];
            int indexdedBytes = Math.Abs(outputStride) * h;
            byte[] indexedValues = new byte[indexdedBytes];
          
                System.Runtime.InteropServices.Marshal.Copy(bmpData.Scan0, rgbValues, 0, bmpBytes);

                if (bytesPerPixel == 3)
                {
                    //Convert the pixel to it's luminance using the formula:
                    // L = .299*R + .587*G + .114*B
                    //Note that ic is the input column and oc is the output column
                    for (r = 0; r < h; r++)
                        for (ic = oc = 0; oc < w; ic += 3, ++oc)
                            indexedValues[r * outputStride + oc] = (byte)(int)
                                (0.299f * rgbValues[r * bmpStride + ic] +
                                 0.587f * rgbValues[r * bmpStride + ic + 1] +
                                 0.114f * rgbValues[r * bmpStride + ic + 2]);
                }
                else //bytesPerPixel == 4
                {
                    //Convert the pixel to it's luminance using the formula:
                    // L = alpha * (.299*R + .587*G + .114*B)
                    //Note that ic is the input column and oc is the output column
                    for (r = 0; r < h; r++)
                        for (ic = oc = 0; oc < w; ic += 4, ++oc)
                            indexedValues[r * outputStride + oc] = (byte)(int)
                                ((rgbValues[r * bmpStride + ic] / 255.0f) *
                                (0.299f * rgbValues[r * bmpStride + ic + 1] +
                                 0.587f * rgbValues[r * bmpStride + ic + 2] +
                                 0.114f * rgbValues[r * bmpStride + ic + 3]));
                }
            
            //Copy the data from the byte array into BitmapData.Scan0
            System.Runtime.InteropServices.Marshal.Copy(indexedValues, 0, outputData.Scan0, indexedValues.Length);
            //Unlock the images
            original.UnlockBits(bmpData);
            output.UnlockBits(outputData);
            return output;
        }
        private Bitmap ResizeBitmap(Bitmap original, int newWidth, int newHeight)
        {
            //a holder for the result
            Bitmap result = new Bitmap(newWidth, newHeight);

            //use a graphics object to draw the resized image into the bitmap
            using (Graphics graphics = Graphics.FromImage(result))
            {
                //set the resize quality modes to high quality
                graphics.CompositingQuality = System.Drawing.Drawing2D.CompositingQuality.HighQuality;
                graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
                graphics.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality;
                //draw the image into the target bitmap
                graphics.DrawImage(original, 0, 0, result.Width, result.Height);
            }

            //return the resulting bitmap
            return result;
        }
        private Bitmap DrawBitmap(double ratio)
        {

            int icolor = 1;
            Size size = new Size(unipien.Bottom.X - unipien.Top.X, unipien.Bottom.Y - unipien.Top.Y);
            int penWidth = (int)Math.Floor((double)size.Height/15);
            Bitmap bmp = new Bitmap(size.Width + penWidth * 2, size.Height + penWidth * 2, PixelFormat.Format24bppRgb);
            Graphics graphicGraph = Graphics.FromImage(bmp);
            //draw bitmap:
            Brush brush = new SolidBrush(Color.White);
            Pen colorPen = new Pen(brush, penWidth);
            graphicGraph.FillRectangle(brush, new Rectangle(0, 0, bmp.Width, bmp.Height));
            foreach (var pt in unipien.PenTrajectories)
            {
                if (pt.Points.Count > 0)
                {

                    Point prepoint = pt.Points[0];
                    prepoint.X = prepoint.X - unipien.Top.X + penWidth;
                    prepoint.Y = unipien.Bottom.Y - prepoint.Y + penWidth;
                    foreach (var point in pt.Points)
                    {
                        brush = new SolidBrush(colorList[icolor]);
                        colorPen = new Pen(brush, penWidth);
                        int x = point.X - unipien.Top.X + penWidth;
                        int y = unipien.Bottom.Y - point.Y + penWidth;
                        Point p = new Point(x, y);

                        graphicGraph.DrawLine(colorPen, prepoint, p);
                        //graphicGraph.FillPie(brush, x, y, penwidth/8, penwidth/8, 0, 360);
                        prepoint = p;
                    }
                    if (icolor < colorList.Count - 1)
                    {
                        icolor++;
                    }
                    else
                    {
                        icolor = 0;
                    }

                }

            }
            int w = (int)Math.Floor(bmp.Width * ratio);
            int h = (int)Math.Floor(bmp.Height * ratio);
            bmp = ResizeBitmap(bmp, w, h);
            return bmp;
        }
        //draw bitmap for grayscale filter
        private Bitmap DrawBitmap(double ratio, int penWidth)
        {

            int icolor = 1;
            Size size = new Size(unipien.Bottom.X - unipien.Top.X, unipien.Bottom.Y - unipien.Top.Y);
            int w = (int)Math.Floor(size.Width * ratio);
            int h = (int)Math.Floor(size.Height * ratio);
            Bitmap bmp = new Bitmap(w + penWidth * 2, h + penWidth * 2, PixelFormat.Format24bppRgb);
            Graphics graphicGraph = Graphics.FromImage(bmp);
            //draw bitmap:
            Brush brush = new SolidBrush(Color.White);
            Pen colorPen = new Pen(brush, penWidth);
            graphicGraph.FillRectangle(brush, new Rectangle(0, 0, bmp.Width, bmp.Height));
            foreach (var pt in unipien.PenTrajectories)
            {
                if (pt.Points.Count > 0)
                {

                    Point prepoint = pt.Points[0];
                    prepoint.X = (int)Math.Floor((prepoint.X - unipien.Top.X) * ratio) + penWidth;
                    prepoint.Y = (int)Math.Floor((unipien.Bottom.Y - prepoint.Y) * ratio) + penWidth;
                    foreach (var point in pt.Points)
                    {
                        brush = new SolidBrush(colorList[icolor]);
                        colorPen = new Pen(brush, penWidth);
                        int x = (int)Math.Floor(((point.X - unipien.Top.X) * ratio)) + penWidth;
                        int y = (int)Math.Floor((unipien.Bottom.Y - point.Y) * ratio) + penWidth;
                        Point p = new Point(x, y);

                        graphicGraph.DrawLine(colorPen, prepoint, p);
                        //graphicGraph.FillPie(brush, x, y, penwidth/8, penwidth/8, 0, 360);
                        prepoint = p;
                    }
                    if (icolor < colorList.Count - 1)
                    {
                        icolor++;
                    }
                    else
                    {
                        icolor = 0;
                    }

                }

            }

            return bmp;
        }

        #endregion

        #region public methods

        public Bitmap GetBitmap(int penWidth, Size bitmapsize, bool isGrayScale)
        {
            Size size = new Size(unipien.Bottom.X - unipien.Top.X + 1, unipien.Bottom.Y - unipien.Top.Y + 1);

            //check if bitmapsize is empty
            if (bitmapsize == Size.Empty)
            {
                bitmapsize = size;
            }
            PixelFormat pf = PixelFormat.Format24bppRgb;
            Bitmap bmp = new Bitmap(bitmapsize.Width, bitmapsize.Height, pf);
            Graphics graphicGraph = Graphics.FromImage(bmp);
            SolidBrush brush = new SolidBrush(Color.White);
            double wRatio = (double)size.Width / (bmp.Width - penWidth * 2);
            double hRatio = (double)size.Height / (bmp.Height - penWidth * 2);

            try
            {
                //draw bmp background
                graphicGraph.FillRectangle(brush, new Rectangle(0, 0, bmp.Width, bmp.Height));
                if (unipien.PenTrajectories.Count > 0)
                {

                    if (wRatio >= hRatio)
                    {
                        Bitmap newbitmap = DrawBitmap(4 / wRatio, penWidth * 4);
                        newbitmap = ResizeBitmap(newbitmap, newbitmap.Width / 4, newbitmap.Height / 4);
                        //Bitmap newbitmap = DrawBitmap(1 / wRatio);
                        graphicGraph.DrawImage(newbitmap, new Point(0, (bmp.Height - newbitmap.Height) / 2));
                    }
                    else
                    {
                        Bitmap newbitmap = DrawBitmap(4 / hRatio, penWidth * 4);
                        newbitmap = ResizeBitmap(newbitmap, newbitmap.Width / 4, newbitmap.Height / 4);
                        //Bitmap newbitmap = DrawBitmap(1 / hRatio);
                        graphicGraph.DrawImage(newbitmap, new Point((bmp.Width - newbitmap.Width) / 2, 0));

                    }
                    //check if grayscale = true
                    if (isGrayScale)
                    {
                        //convert bitmap to grayscale 8bit
                        bmp = ColorToGrayscale(bmp);
                    }


                }
            }
            catch (Exception ex)
            {
                return null;
            }
            return bmp;

        }
        public byte[] ConvertGrayscaleBitmaptoBytes(Bitmap original)
        {
            Bitmap gsBitmap = (Bitmap)original.Clone();
            if (original.PixelFormat != PixelFormat.Format8bppIndexed)
            {
                gsBitmap = ColorToGrayscale(original);
            }

            byte[] grayscalebytes = new byte[gsBitmap.Width * gsBitmap.Height];
            BitmapData bmpData = gsBitmap.LockBits(new Rectangle(0, 0, gsBitmap.Width, gsBitmap.Height),
                                                ImageLockMode.ReadOnly,
                                                gsBitmap.PixelFormat);
            int bytes = Math.Abs(bmpData.Stride) * gsBitmap.Height;
            byte[] rgbValues = new byte[bytes];

            System.Runtime.InteropServices.Marshal.Copy(bmpData.Scan0, rgbValues, 0, bytes);
            gsBitmap.UnlockBits(bmpData);
            //
            for (int h = 0; h < gsBitmap.Height; h++)
            {
                for (int w = 0; w < gsBitmap.Width; w++)
                {
                    grayscalebytes[h * gsBitmap.Width + w] = rgbValues[h * bmpData.Stride + w];
                }
            }

            return grayscalebytes;

        }

        #endregion

    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)


Written By
Vietnam Maritime University
Vietnam Vietnam
This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Comments and Discussions