Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version

Image Tracking and Computer Vision Using Fourier Image Correlation

, 23 Apr 2013 CPOL
How to teach a program to recognize something within a video stream.
OpenCVDemo3-noexe.zip
OpenCVDemo
Description.odt
OpenCvSharp-2.3.1-x86-20111030
XmlDoc-English
XmlDoc-Japanese
TrackingEyes.jpg
WindowsFormsApplication1
Properties
Settings.settings
OpenCVDemo3.zip
Description.odt
OpenCv-2.3.1-x86
opencv_calib3d231.dll
opencv_contrib231.dll
opencv_core231.dll
opencv_features2d231.dll
opencv_ffmpeg.dll
opencv_flann231.dll
opencv_gpu231.dll
opencv_highgui231.dll
opencv_imgproc231.dll
opencv_legacy231.dll
opencv_ml231.dll
opencv_objdetect231.dll
opencv_ts231.dll
opencv_video231.dll
tbb.dll
OpenCvSharp.Blob.dll
OpenCvSharp.CPlusPlus.dll
OpenCvSharp.DebuggerVisualizers.dll
OpenCvSharp.dll
OpenCvSharp.Extensions.dll
OpenCvSharp.MachineLearning.dll
OpenCvSharp.UserInterface.dll
OpenCvSharpExtern.dll
TrackingEyes.jpg
Settings.settings
///////////////////////////////////////////////////////////////////////////////
//
//  TrackingUI.cs
//
//  By Philip R. Braica (HoshiKata@aol.com, VeryMadSci@gmail.com)
//
//  Distributed under the The Code Project Open License (CPOL)
//  http://www.codeproject.com/info/cpol10.aspx
///////////////////////////////////////////////////////////////////////////////

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Drawing;
using System.Data;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using OpenCvSharp;

namespace OpenCVDemo
{
    /// <summary>
    /// Demo UI for tracking with open CV.
    /// </summary>
    public partial class TrackingUI : UserControl
    {
        /// <summary>
        /// Constructor.
        /// </summary>
        public TrackingUI()
        {
            InitializeComponent();
            pictureBox1.MouseDown += new MouseEventHandler(pictureBox1_MouseDown);
            pictureBox1.MouseUp += new MouseEventHandler(pictureBox1_MouseUp);
            pictureBox1.MouseMove += new MouseEventHandler(pictureBox1_MouseMove);
            button2.Enabled = false;
            button3.Enabled = false;
            button4.Enabled = false;
            button5.Enabled = false;
            button6.Enabled = false;
            
        }

        /// <summary>
        /// Load video
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button1_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
            button7.Enabled = false;
            OpenFileDialog ofd = new OpenFileDialog();
            DialogResult dr = ofd.ShowDialog();
            if (dr != DialogResult.OK) return;

            if (m_cvCapture != null)
            {
                m_cvCapture.Dispose();
            }
            m_cvCapture = CvCapture.FromFile(ofd.FileName);
            m_videoFileName = ofd.FileName;

            // Hack to get first frame.
            timer1_Tick(this, new EventArgs());
            if (m_leftFront != null)
            {
                m_firstFrame = new Bitmap(m_leftFront);
            }

            // Start playing.
            timer1.Enabled = true;
            button7.Enabled = true;
            button2.Enabled = false;
            button4.Enabled = false;
            button5.Enabled = false;
            button6.Enabled = true;
            button3.Enabled = true;
        }

        #region Selecting the rectangle.
        /// <summary>
        /// Select rect.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button3_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
            button7.Enabled = false;
            pictureBox1.Image = m_firstFrame;
            m_selectingRect = true;
        }

        /// <summary>
        /// Mouse up selected.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        void pictureBox1_MouseUp(object sender, MouseEventArgs e)
        {
            if (!m_selectingRect) return;
            float x = e.X - pictureBox1.Left;
            float y = e.Y - pictureBox1.Top;
            float iw = pictureBox1.Image.Width;
            float ih = pictureBox1.Image.Height;
            float w = pictureBox1.Width;
            float h = pictureBox1.Height;
            int nx = (int)(x * iw / w);
            int ny = (int)(y * ih / h);
            int ox = m_selectedRect.Location.X;
            int oy = m_selectedRect.Location.Y;
            if (nx < ox)
            {
                if (ny < oy) m_selectedRect = new Rectangle(nx, ny, ox - nx, oy - ny);
                else m_selectedRect = new Rectangle(nx, oy, ox - nx, ny - oy);
            }
            else
            {
                if (ny < oy) m_selectedRect = new Rectangle(ox, ny, nx - ox, oy - ny);
                else m_selectedRect = new Rectangle(ox, oy, nx - ox, ny - oy);
            }
            m_selectingRect = false;
            if ((m_selectedRect.Width > 4) && (m_selectedRect.Height > 4))
            {
                button2.Enabled = true;
                button4.Enabled = true;
                button5.Enabled = true;
                button6.Enabled = true;
            }
        }

        /// <summary>
        /// Mouse moved.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        void pictureBox1_MouseMove(object sender, MouseEventArgs e)
        {
            if (!m_selectingRect) return;
            float x = e.X - pictureBox1.Left;
            float y = e.Y - pictureBox1.Top;
            float iw = pictureBox1.Image.Width;
            float ih = pictureBox1.Image.Height;
            float w = pictureBox1.Width;
            float h = pictureBox1.Height;
            int nx = (int)(x * iw / w);
            int ny = (int)(y * ih / h);
            int ox = m_selectedRect.Location.X;
            int oy = m_selectedRect.Location.Y;
            if (nx < ox)
            {
                if (ny < oy) m_selectedRect = new Rectangle(nx, ny, ox - nx, oy - ny);
                else m_selectedRect = new Rectangle(nx, oy, ox - nx, ny - oy);
            }
            else
            {
                if (ny < oy) m_selectedRect = new Rectangle(ox, ny, nx - ox, oy - ny);
                else m_selectedRect = new Rectangle(ox, oy, nx - ox, ny - oy);
            }

            setupBuffers(m_firstFrame.Width, m_firstFrame.Height);

            using (Graphics g = Graphics.FromImage(m_leftBack))
            {
                g.DrawImage(m_firstFrame, 0, 0, m_leftBack.Width, m_leftBack.Height);
                using (Pen p = new Pen(Color.FromArgb(128, Color.Green), 4f))
                {
                    if (e.Button == System.Windows.Forms.MouseButtons.None)
                    {
                        Rectangle sp = new Rectangle(nx - 2, ny - 2, 4, 4);
                        g.DrawEllipse(p, sp);
                    }
                    else
                    {
                        g.DrawRectangle(p, m_selectedRect);
                    }
                }
            }
            pictureBox1.Image = m_leftBack;
        }

        /// <summary>
        /// Setup the image buffers.
        /// </summary>
        /// <param name="w"></param>
        /// <param name="h"></param>
        protected void setupBuffers(int w, int h)
        {
            bool doSetup = true;
            if (m_leftBack != null)
            {
                if ((m_leftBack.Width == w) || (m_leftBack.Height == h))
                {
                    doSetup = false;
                }
            }
            if (doSetup)
            {
                if (m_leftBack != null) m_leftBack.Dispose();
                if (m_rightBack != null) m_rightBack.Dispose();
                m_leftBack = new System.Drawing.Bitmap(w, h, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
                m_rightBack = new System.Drawing.Bitmap(w, h, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
            }
        }

        /// <summary>
        /// Down.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        void pictureBox1_MouseDown(object sender, MouseEventArgs e)
        {
            if (!m_selectingRect) return; 
            float x = e.X - pictureBox1.Left;
            float y = e.Y - pictureBox1.Top;
            float iw = pictureBox1.Image.Width;
            float ih = pictureBox1.Image.Height;
            float w = pictureBox1.Width;
            float h = pictureBox1.Height;

            m_selectedRect.Location = new Point((int)(x * iw / w), (int)(y * ih / h));
        }
        #endregion

        /// <summary>
        /// Learn.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button2_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
            button7.Enabled = false;
            if (m_cvCapture != null)
            {
                m_cvCapture.Dispose();
            }
            m_cvCapture = CvCapture.FromFile(m_videoFileName);
            m_learning = true;
            if (m_learn == null)
            {
                m_learn = new Tracker();
                m_learn.Font = this.Font;
            }
            m_learn.Reset();
            if (m_cvVideoWriter != null)
            {
                m_cvVideoWriter.Dispose();
                m_cvVideoWriter = null;
            }
            m_logging = false;
            timer1.Enabled = true;
            button7.Enabled = true;
        }

        /// <summary>
        /// Learn and save.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button4_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
            button7.Enabled = false;
            if (m_cvCapture != null)
            {
                m_cvCapture.Dispose();
            }
            m_cvCapture = CvCapture.FromFile(m_videoFileName);
            m_learning = true;
            SaveFileDialog sfd = new SaveFileDialog();
            DialogResult dr = sfd.ShowDialog();
            if (dr != DialogResult.OK)
            {
                return;
            }
            if (m_cvVideoWriter != null)
            {
                m_cvVideoWriter.Dispose();
                m_cvVideoWriter = null;
            }
            m_logging = false;

            m_cvVideoWriter = new CvVideoWriter(sfd.FileName,
                FourCC.DIB, // IYUB, MSVC, DIB, CVID are supported.
                10,
                new CvSize(m_rightBack.Width, m_rightBack.Height));
            m_learn.Reset();
            timer1.Enabled = true;
            button7.Enabled = true;
        }

        /// <summary>
        /// Learn and log.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button5_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
            button7.Enabled = false;
            if (m_cvCapture != null)
            {
                m_cvCapture.Dispose();
            }
            m_cvCapture = CvCapture.FromFile(m_videoFileName);
            m_learning = true;
            if (m_learn == null)
            {
                m_learn = new Tracker();
            }
            m_learn.Reset();

            SaveFileDialog sfd = new SaveFileDialog();
            DialogResult dr = sfd.ShowDialog();
            if (dr != DialogResult.OK)
            {
                return;
            }
            if (m_cvVideoWriter != null)
            {
                m_cvVideoWriter.Dispose();
                m_cvVideoWriter = null;
            }
            m_logFile = sfd.FileName;
            m_logging = true;
            timer1.Enabled = true;
            button7.Enabled = true;
        }

        #region Protected data.

        /// <summary>
        /// Log file.
        /// </summary>
        protected string m_logFile = "";
        
        /// <summary>
        /// Logging.
        /// </summary>
        protected bool m_logging = false;

        /// <summary>
        /// The selected rectangle.
        /// </summary>
        protected Rectangle m_selectedRect = new Rectangle(0, 0, 0, 0);

        /// <summary>
        /// We are selecting a rectangle.
        /// </summary>
        protected bool m_selectingRect = false;

        /// <summary>
        /// Capture device.
        /// </summary>
        protected CvCapture m_cvCapture = null;

        /// <summary>
        /// Video writer.
        /// </summary>
        protected CvVideoWriter m_cvVideoWriter = null;

        /// <summary>
        /// The name of the last video file loaded.
        /// </summary>
        protected string m_videoFileName = "";

        /// <summary>
        /// Learning flag.
        /// </summary>
        protected bool m_learning = false;

        /// <summary>
        /// First frame.
        /// </summary>
        protected Bitmap m_firstFrame = null;

        /// <summary>
        /// Left back buffer.
        /// </summary>
        protected Bitmap m_leftBack = null;

        /// <summary>
        /// Right back buffer.
        /// </summary>
        protected Bitmap m_rightBack = null;

        /// <summary>
        /// Left front buffer.
        /// </summary>
        protected Bitmap m_leftFront = null;

        /// <summary>
        /// Right front buffer.
        /// </summary>
        protected Bitmap m_rightFront = null;

        /// <summary>
        /// Learning object.
        /// </summary>
        protected Tracker m_learn = null;
        #endregion

        /// <summary>
        /// Learn.
        /// </summary>
        /// <param name="img"></param>
        protected void learn(IplImage img, Bitmap markup)
        {
            if (m_learn == null)
            {
                m_learn = new Tracker();
            }
            if ((m_selectedRect.Width > 3) && (m_selectedRect.Height > 3))
            {
                m_learn.Learn(img, markup, m_selectedRect, m_logging);
            }
        }

        /// <summary>
        /// Save a frame.
        /// </summary>
        /// <param name="img"></param>
        protected void saveFrame(Bitmap img)
        {
            if (img == null) return;
            CvVideoWriter cvw = m_cvVideoWriter;
            if (cvw == null) return;
            if (cvw.IsDisposed) return;
            IplImage ipltmp = IplImage.FromBitmap(img);
            cvw.WriteFrame(ipltmp);
            ipltmp.Dispose();
        }


        /// <summary>
        /// Timer, grabs an input frame if there is one,
        /// triggers processing and handles double buffering as needed.
        /// 
        /// Also uses simple double buffering.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void timer1_Tick(object sender, EventArgs e)
        {
            // This is demo code, so not everything is thread safe but
            // Since the obvious most annoying race is m_cvCapture changing,
            // This made sence to guard against.
            //
            // Grab a local copy of the reference then act on it.
            CvCapture threadSafeCapture = m_cvCapture;
            if (threadSafeCapture == null) return;
            threadSafeCapture.GrabFrame();
            IplImage img = threadSafeCapture.RetrieveFrame();

            // Video is done.
            if ((img == null) || (threadSafeCapture.PosFrames == threadSafeCapture.FrameCount))
            {
                m_cvCapture.Dispose();
                m_cvCapture = null;
                timer1.Enabled = false;
                button7.Enabled = false;
                if (m_cvVideoWriter != null)
                {
                    m_cvVideoWriter.Dispose();
                    m_cvVideoWriter = null;
                }
                if (m_logging)
                {
                    using (System.IO.StreamWriter sw = new System.IO.StreamWriter(m_logFile))
                    {
                        sw.Write(m_learn.MakeLogReport());
                        sw.Close();
                    }
                }
                return;
            }

            // Make sure the bitmap buffers are the right size.
            setupBuffers(img.Width, img.Height);

            // Draw the raw frame to the left.
            System.Drawing.Bitmap tmp = img.ToBitmap();
            using (System.Drawing.Graphics g = System.Drawing.Graphics.FromImage(m_leftBack))
            {
                g.DrawImage(tmp, 0, 0, m_leftBack.Width, m_leftBack.Height);
            }
            using (System.Drawing.Graphics g = System.Drawing.Graphics.FromImage(m_rightBack))
            {
                g.DrawImage(tmp, 0, 0, m_rightBack.Width, m_rightBack.Height);
            }

            // Learn if learning.
            if (m_learning)
            {
                learn(img, m_rightBack);
            }

            // Save if saving.
            if (m_cvVideoWriter != null)
            {
                saveFrame(m_rightBack);
            }
              
            // Dispose of the temp. bitmap.
            tmp.Dispose();


            // Swap left and right buffers and display.
            System.Drawing.Bitmap btmp = m_leftBack;
            m_leftBack = m_leftFront;
            m_leftFront = btmp;
            btmp = m_rightBack;
            m_rightBack = m_rightFront;
            m_rightFront = btmp;
            pictureBox1.Image = m_leftFront;
            pictureBox2.Image = m_rightFront;
        }

        /// <summary>
        /// Save a feature.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button6_Click(object sender, EventArgs e)
        {
            SaveFileDialog sfd = new SaveFileDialog();
            DialogResult dr = sfd.ShowDialog();
            if ((dr == DialogResult.OK) || (dr == DialogResult.Yes))
            {
                m_learn.SaveFeature(sfd.FileName, textBox1.Text);
            }
        }

        /// <summary>
        /// Stop the timer.
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button7_Click(object sender, EventArgs e)
        {
            timer1.Enabled = false;
        }



    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

HoshiKata
Software Developer (Senior) KMC Systems
United States United States
Phil is a Principal Software developer focusing on weird yet practical algorithms that run the gamut of embedded and desktop (PID loops, Kalman filters, FFTs, client-server SOAP bindings, ASIC design, communication protocols, game engines, robotics).
 
In his personal life he is a part time mad scientist, full time dad, and studies small circle jujitsu, plays guitar and piano.

| Advertise | Privacy | Mobile
Web02 | 2.8.141022.1 | Last Updated 23 Apr 2013
Article Copyright 2012 by HoshiKata
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid