Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version

Image Tracking and Computer Vision Using Fourier Image Correlation

, 23 Apr 2013
How to teach a program to recognize something within a video stream.
OpenCVDemo3-noexe.zip
OpenCVDemo
Description.odt
OpenCvSharp-2.3.1-x86-20111030
XmlDoc-English
XmlDoc-Japanese
TrackingEyes.jpg
WindowsFormsApplication1
Properties
Settings.settings
OpenCVDemo3.zip
Description.odt
OpenCv-2.3.1-x86
opencv_calib3d231.dll
opencv_contrib231.dll
opencv_core231.dll
opencv_features2d231.dll
opencv_ffmpeg.dll
opencv_flann231.dll
opencv_gpu231.dll
opencv_highgui231.dll
opencv_imgproc231.dll
opencv_legacy231.dll
opencv_ml231.dll
opencv_objdetect231.dll
opencv_ts231.dll
opencv_video231.dll
tbb.dll
OpenCvSharp.Blob.dll
OpenCvSharp.CPlusPlus.dll
OpenCvSharp.DebuggerVisualizers.dll
OpenCvSharp.dll
OpenCvSharp.Extensions.dll
OpenCvSharp.MachineLearning.dll
OpenCvSharp.UserInterface.dll
OpenCvSharpExtern.dll
TrackingEyes.jpg
Settings.settings
///////////////////////////////////////////////////////////////////////////////
//
//  Feature.cs
//
//  By Philip R. Braica (HoshiKata@aol.com, VeryMadSci@gmail.com)
//
//  Distributed under the The Code Project Open License (CPOL)
//  http://www.codeproject.com/info/cpol10.aspx
///////////////////////////////////////////////////////////////////////////////

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using OpenCvSharp;
using System.Drawing;
using System.Drawing.Imaging;

namespace OpenCVDemo
{
    /// <summary>
    /// A feature is a set of targets to check for and a way to manage them.
    /// </summary>
    [Serializable()]
    public class Feature
    {
        /// <summary>
        /// Target collection name.
        /// </summary>
        public string Name { get; set; }

        /// <summary>
        /// The target set.
        /// </summary>
        [System.Xml.Serialization.XmlArray]
        public List<Target> Targets = new List<Target>();

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float AutoCorrelationThreshold { get; set; }

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float LearningCorrelationThreshold { get; set; }

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float CrossCorrelationThreshold { get; set; }

        /// <summary>
        /// Default constructor.
        /// </summary>
        public Feature()
        {
        }

        /// <summary>
        /// Constructor with variables.
        /// </summary>
        /// <param name="targets"></param>
        /// <param name="auto"></param>
        /// <param name="learn"></param>
        /// <param name="cross"></param>
        public Feature(string name, List<Target> targets, float auto, float learn, float cross)
        {
            Setup(name, targets, auto, learn, cross);
        }

        /// <summary>
        /// Setup
        /// </summary>
        /// <param name="targets"></param>
        /// <param name="auto"></param>
        /// <param name="learn"></param>
        /// <param name="cross"></param>
        public void Setup(string name, List<Target> targets, float auto, float learn, float cross)
        {
            Name = name;
            Targets.Clear();
            Targets.AddRange(targets);
            AutoCorrelationThreshold = auto;
            LearningCorrelationThreshold = learn;
            CrossCorrelationThreshold = cross;
        }

        /// <summary>
        /// Save to a file.
        /// </summary>
        /// <param name="fileName"></param>
        public void Save(string fileName)
        {
            System.Xml.Serialization.XmlSerializer xmls =
                   new System.Xml.Serialization.XmlSerializer(this.GetType());

            using (System.IO.FileStream fs = new System.IO.FileStream(fileName,
                    System.IO.FileMode.OpenOrCreate,
                    System.IO.FileAccess.Write,
                    System.IO.FileShare.ReadWrite))
            {
                xmls.Serialize(fs, this);
            }
        }

        /// <summary>
        /// Load from XML file.
        /// </summary>
        /// <param name="fileName"></param>
        public void Load(string fileName)
        {
            System.Xml.Serialization.XmlSerializer xmls =
                new System.Xml.Serialization.XmlSerializer(this.GetType());
            using (System.IO.TextReader tr = new System.IO.StreamReader(fileName))
            {
                Feature tc = (Feature)xmls.Deserialize(tr);
                Setup(tc.Name, tc.Targets, tc.AutoCorrelationThreshold, tc.LearningCorrelationThreshold, tc.CrossCorrelationThreshold);
                tr.Close();
            }
        }

        /// <summary>
        /// Setup
        /// </summary>
        /// <param name="targets"></param>
        /// <param name="auto"></param>
        /// <param name="learn"></param>
        /// <param name="cross"></param>
        public void Restore(Tracker learning)
        {
            learning.AutoCorrelationThreshold = AutoCorrelationThreshold;
            learning.LearningCorrelationThreshold = LearningCorrelationThreshold;
            learning.CrossCorrelationThreshold = CrossCorrelationThreshold;
            learning.SetTargets(Targets);
        }
    }

    /// <summary>
    /// Target class.
    /// </summary>
    [Serializable()]
    public class Target
    {
        /// <summary>
        /// All scores that are recorded.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        public List<float> Scores = new List<float>();

        /// <summary>
        /// Cross correlation scores.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        public List<float> CrossScores = new List<float>();

        /// <summary>
        /// The target.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        public IplImage SearchTarget = null;

        /// <summary>
        /// Last correlation value.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        public float LastCorrelation = 0;

        /// <summary>
        /// Set to false to disable.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public bool Enabled { get; set; }

        /// <summary>
        /// Last position.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        public Rectangle LastRect = new Rectangle();

        /// <summary>
        /// The scratch pad.
        /// </summary>
        [System.Xml.Serialization.XmlIgnore]
        protected IplImage m_scratch = null;

        /// <summary>
        /// Bin hex of the image.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public string Base64
        {
            get
            {
                if (SearchTarget != null)
                {
                    using (Bitmap b = SearchTarget.ToBitmap())
                    {
                        using (System.IO.MemoryStream ms = new System.IO.MemoryStream())
                        {
                            b.Save(ms, System.Drawing.Imaging.ImageFormat.Bmp);
                            byte[] bytes = ms.ToArray();
                            return Convert.ToBase64String(bytes);
                        }
                    }
                }
                return "";
            }
            set
            {
                byte[] bytes = Convert.FromBase64String(value);
                using (System.IO.MemoryStream ms = new System.IO.MemoryStream(bytes, 0, bytes.Length))
                {
                    ms.Write(bytes, 0, bytes.Length);
                    using (Bitmap b = new Bitmap(ms, true))
                    {
                        IplImage old = SearchTarget;
                        SearchTarget = IplImage.FromBitmap(b);
                        if (old != null) old.Dispose();
                    }
                }
            }
        }

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float AutoCorrelationThreshold { get; set; }

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float LearningCorrelationThreshold { get; set; }

        /// <summary>
        /// The auto correlation threshold.
        /// </summary>
        [System.Xml.Serialization.XmlElement]
        public float CrossCorrelationThreshold { get; set; }

        /// <summary>
        /// So we can latter just add XML markups with attributes have a default constructor.
        /// Of course we'd also have to convert over to mostly properties.
        /// </summary>
        public Target()
        {
            Enabled = true;
        }

        /// <summary>
        /// Constructor w/ image and rectangle.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="r"></param>
        public Target(IplImage img, Rectangle r)
        {
            SearchTarget = img.GetSubImage(new CvRect(r.X, r.Y, r.Width, r.Height));
            Enabled = true;
        }

        /// <summary>
        /// Dispose.
        /// </summary>
        public void Dispose()
        {
            if (SearchTarget != null) { SearchTarget.Dispose(); SearchTarget = null; }
        }

        /// <summary>
        /// Cross correlate.
        /// </summary>
        /// <param name="foundLocation"></param>
        public virtual void CrossCorrelate(Rectangle foundLocation)
        {
            if (m_scratch == null) return;
            int crossDiameter = 40;
            for (int w = -crossDiameter; w < foundLocation.Width + crossDiameter; w++)
            {
                for (int h = -crossDiameter; h < foundLocation.Height + crossDiameter; h++)
                {
                    int x = w + foundLocation.X;
                    x = x < 0 ? 0 : x >= m_scratch.Width ? m_scratch.Width - 1 : x;
                    int y = h + foundLocation.Y;
                    y = y < 0 ? 0 : y >= m_scratch.Height ? m_scratch.Height - 1 : y;

                    m_scratch[y, x] = 0;
                }
            }

            CvPoint minloc = new CvPoint();
            CvPoint maxloc = new CvPoint();
            double minv = 0, maxv = 0;

            m_scratch.MinMaxLoc(out minv, out maxv, out minloc, out maxloc);
            CrossScores.Add((float)maxv);
        }

        /// <summary>
        /// Compute the correlation of image A and B fourier style.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="target"></param>
        /// <param name="scratchImage"></param>
        public virtual float Correlate(IplImage img, CvRect searchRect)
        {
            img.ResetROI();
            img.SetROI(searchRect);

            int desiredWidth = (int)(searchRect.Width - SearchTarget.Width + 1);
            int desiredHeight = (int)(searchRect.Height - SearchTarget.Height + 1);
            if ((desiredWidth < 0) || (desiredHeight < 0))
            {
                return 0;
            }
            if (m_scratch == null)
            {
                m_scratch = new IplImage(desiredWidth, desiredHeight, BitDepth.F32, 1);
            }
            else
            {
                if ((m_scratch.Width != desiredWidth) || (m_scratch.Height != desiredHeight))
                {
                    m_scratch.Dispose();
                    m_scratch = new IplImage(desiredWidth, desiredHeight, BitDepth.F32, 1);
                }
            }
            m_scratch.Zero();
            try
            {
                img.MatchTemplate(SearchTarget, m_scratch, MatchTemplateMethod.CCoeffNormed);
            }
            catch (Exception)
            {
                return 0;
            }
            img.ResetROI();

            CvPoint minloc = new CvPoint();
            CvPoint maxloc = new CvPoint();
            double minv = 0, maxv = 0;

            m_scratch.MinMaxLoc(out minv, out maxv, out minloc, out maxloc);

            maxloc.X += searchRect.X;
            maxloc.Y += searchRect.Y;
            LastRect = new Rectangle(maxloc.X, maxloc.Y, SearchTarget.Width, SearchTarget.Height);
            LastCorrelation = (float)maxv;
            Scores.Add(LastCorrelation);
            return (float)maxv;
        }


        /// <summary>
        /// Make a list of all locations with something greater than the threshold.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="searchRect"></param>
        /// <param name="minimum"></param>
        /// <param name="maxRects"></param>
        /// <returns></returns>
        public virtual List<Rectangle> Find(IplImage img, CvRect searchRect, float minimum, int maxRects)
        {
            img.ResetROI();
            img.SetROI(searchRect);

            int desiredWidth = (int)(img.ROI.Width - SearchTarget.Width + 1);
            int desiredHeight = (int)(img.ROI.Height - SearchTarget.Height + 1);
            if ((desiredWidth < 0) || (desiredHeight < 0))
            {
                return null;
            }
            if (m_scratch == null)
            {
                m_scratch = new IplImage(desiredWidth, desiredHeight, BitDepth.F32, 1);
            }
            else
            {
                if ((m_scratch.Width != desiredWidth) || (m_scratch.Height != desiredHeight))
                {
                    m_scratch.Dispose();
                    m_scratch = new IplImage(desiredWidth, desiredHeight, BitDepth.F32, 1);
                }
            }
            m_scratch.Zero();
            try
            {
                img.MatchTemplate(SearchTarget, m_scratch, MatchTemplateMethod.CCoeffNormed);
            }
            catch (Exception)
            {
                return null;
            }
            img.ResetROI();

            List<Rectangle> rv = new List<Rectangle>();
            double detectionValue = minimum + 1;
            while ((detectionValue > minimum) && (rv.Count < maxRects))
            {
                CvPoint minloc = new CvPoint();
                CvPoint maxloc = new CvPoint();
                double minv = 0;

                m_scratch.MinMaxLoc(out minv, out detectionValue, out minloc, out maxloc);

                maxloc.X += searchRect.X;
                maxloc.Y += searchRect.Y;
                LastRect = new Rectangle(maxloc.X, maxloc.Y, SearchTarget.Width, SearchTarget.Height);
                LastCorrelation = (float)detectionValue;

                int minPixels = 10;
                if (detectionValue > minimum)
                {
                    rv.Add(LastRect);
                    int xc = LastRect.X;
                    int yc = LastRect.Y;
                    int xmin = xc - (LastRect.Width / 2) - minPixels;
                    int xmax = xmin + LastRect.Width + minPixels + minPixels;
                    int ymin = yc - (LastRect.Height / 2) - minPixels;
                    int ymax = ymin + LastRect.Height + minPixels + minPixels;
                    xmin = xmin < 0 ? 0 : xmin;
                    ymin = ymin < 0 ? 0 : ymin;
                    int xmaxs = xmax > m_scratch.Width - 1 ? m_scratch.Width - 1 : xmax;
                    int ymaxs = ymax > m_scratch.Height - 1 ? m_scratch.Height - 1 : ymax;
                    xmax = xmax > img.Width - 1 ? img.Width - 1 : xmax;
                    ymax = ymax > img.Height - 1 ? img.Height - 1 : ymax;

                    for (int x = xmin; x < xmaxs; x++)
                    {
                        for (int y = ymin; y < ymaxs; y++)
                        {
                            m_scratch[y, x] = 0;
                        }
                    }

                    // adjust for number of channels.
                    for (int x = xmin; x < xmax; x++)
                    {
                        for (int y = ymin; y < ymax; y++)
                        {
                            img[y, x] = 0;
                        }
                    }

                    
                }
            }

            return rv;
        }
    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

HoshiKata
Software Developer (Senior) KMC Systems
United States United States
Phil is a Principal Software developer focusing on weird yet practical algorithms that run the gamut of embedded and desktop (PID loops, Kalman filters, FFTs, client-server SOAP bindings, ASIC design, communication protocols, game engines, robotics).
 
In his personal life he is a part time mad scientist, full time dad, and studies small circle jujitsu, plays guitar and piano.

| Advertise | Privacy | Mobile
Web03 | 2.8.140814.1 | Last Updated 23 Apr 2013
Article Copyright 2012 by HoshiKata
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid