Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version

Image Tracking and Computer Vision Using Fourier Image Correlation

, 23 Apr 2013 CPOL
How to teach a program to recognize something within a video stream.
OpenCVDemo3-noexe.zip
OpenCVDemo
Description.odt
OpenCvSharp-2.3.1-x86-20111030
XmlDoc-English
XmlDoc-Japanese
TrackingEyes.jpg
WindowsFormsApplication1
Properties
Settings.settings
OpenCVDemo3.zip
Description.odt
OpenCv-2.3.1-x86
opencv_calib3d231.dll
opencv_contrib231.dll
opencv_core231.dll
opencv_features2d231.dll
opencv_ffmpeg.dll
opencv_flann231.dll
opencv_gpu231.dll
opencv_highgui231.dll
opencv_imgproc231.dll
opencv_legacy231.dll
opencv_ml231.dll
opencv_objdetect231.dll
opencv_ts231.dll
opencv_video231.dll
tbb.dll
OpenCvSharp.Blob.dll
OpenCvSharp.CPlusPlus.dll
OpenCvSharp.DebuggerVisualizers.dll
OpenCvSharp.dll
OpenCvSharp.Extensions.dll
OpenCvSharp.MachineLearning.dll
OpenCvSharp.UserInterface.dll
OpenCvSharpExtern.dll
TrackingEyes.jpg
Settings.settings
///////////////////////////////////////////////////////////////////////////////
//
//  Classifier.cs
//
//  By Philip R. Braica (HoshiKata@aol.com, VeryMadSci@gmail.com)
//
//  Distributed under the The Code Project Open License (CPOL)
//  http://www.codeproject.com/info/cpol10.aspx
///////////////////////////////////////////////////////////////////////////////

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using OpenCvSharp;
using System.Drawing;
using System.Drawing.Imaging;

namespace OpenCVDemo
{
    /// <summary>
    /// A classifier is a list of features to check for in an image.
    /// </summary>
    public class Classifier
    {
        /// <summary>
        /// The features.
        /// </summary>
        public List<Feature> Features = new List<Feature>();

        /// <summary>
        /// The font to use.
        /// </summary>
        public Font Font = new Font("Microsoft Sans Serif", 8.00f);

        /// <summary>
        /// Add a feature to the list of features to track.
        /// </summary>
        /// <param name="fileName"></param>
        public void AppendFeature(string fileName)
        {
            Feature f = new Feature();
            f.Load(fileName);
            for (int i = 0; i < f.Targets.Count; i++)
            {
                if (f.Targets[i].Enabled == false)
                {
                    f.Targets.RemoveAt(i);
                }
            }
            Features.Add(f);
        }

        /// <summary>
        /// Rset all but features.
        /// </summary>
        public void Reset()
        {
            searchState = 0;
            Trackers.Clear();
        }

        /// <summary>
        /// Search state.
        /// </summary>
        protected int searchState = 0;

        /// <summary>
        /// Trackers
        /// </summary>
        protected List<AssociativeTracker> Trackers = new List<AssociativeTracker>();

        /// <summary>
        /// Track an object or objects in frames based on target array.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="imgOut"></param>
        /// <param name="r"></param>
        public void Classify(IplImage img, Bitmap imgOut)
        {
            CvRect searchRect = new CvRect(0, 0, img.Width, img.Height);
            int detections = 0;

            if (((searchState == 0) || (Trackers.Count == 0)) && (Trackers.Count < 10))
            {
                // finding things, first for ideals
                for (int i = 0; i < Features.Count; i++)
                {
                    for (int j = 0; j < Features[i].Targets.Count; j++)
                    {
                        if (!Features[i].Targets[j].Enabled) continue;
                        List<Rectangle> rects = Features[i].Targets[j].Find(
                            img,
                            searchRect,
                            Features[i].AutoCorrelationThreshold,
                            10);
                        if (rects == null) continue;
                        for (int k = 0; k < rects.Count; k++)
                        {
                            marklocation(imgOut, Features[i].Name, Features[i].Targets[j], rects[k], 0);
                            Trackers.Add(new AssociativeTracker(Features[i], j, rects[k]));
                            detections++;
                        }
                    }
                    if (detections > 30)
                    {
                        return;
                    }
                }

                // Next not ideals.
                for (int i = 0; i < Features.Count; i++)
                {
                    for (int j = 0; j < Features[i].Targets.Count; j++)
                    {
                        if (!Features[i].Targets[j].Enabled) continue;
                        List<Rectangle> rects = Features[i].Targets[j].Find(
                            img,
                            searchRect,
                            Features[i].CrossCorrelationThreshold,
                            10);
                        if (rects == null) continue;
                        for (int k = 0; k < rects.Count; k++)
                        {
                            marklocation(imgOut, Features[i].Name, Features[i].Targets[j], rects[k], 0);
                            Trackers.Add(new AssociativeTracker(Features[i], j, rects[k]));
                            detections++;
                        }
                    }
                    if (detections > 30)
                    {
                        return;
                    }
                }
            }
            else
            {
                for (int i = 0; i < Trackers.Count; i++)
                {
                    Trackers[i].FoundThisPass = false;
                }
                for (int i = 0; i < Trackers.Count; i++)
                {
                    if (Trackers[i].FirstPass(img, imgOut))
                    {
                        marklocation(
                            imgOut,
                            Trackers[i].Feature.Name,
                            Trackers[i].Feature.Targets[Trackers[i].TargetIndex],
                            Trackers[i].Position.Bounds,
                            i + 1);
                    }
                }

                for (int i = 0; i < Trackers.Count; i++)
                {
                    if (Trackers[i].FoundThisPass) continue;
                    if (Trackers[i].SecondPass(img, imgOut))
                    {
                        marklocation(
                            imgOut,
                            Trackers[i].Feature.Name,
                            Trackers[i].Feature.Targets[Trackers[i].TargetIndex],
                            Trackers[i].Position.Bounds,
                            i + 1);
                    }
                }
               /* for (int i = 0; i < Trackers.Count; i++)
                {
                    if (Trackers[i].FoundThisPass) continue;
                    if (Trackers[i].ThirdPass(img, imgOut))
                    {
                        marklocation(
                            imgOut,
                            Trackers[i].Feature.Name,
                            Trackers[i].Feature.Targets[Trackers[i].TargetIndex],
                            Trackers[i].Position.Bounds,
                            i + 1);
                    }
                } */
                for (int i = 0; i < Trackers.Count; i++)
                {
                    if (!Trackers[i].FoundThisPass)
                    {
                        if (Trackers[i].FoundCount > 0)
                        {
                            Trackers[i].FoundCount = 0;
                        }
                        else
                        {
                            Trackers[i].FoundCount--;
                        }
                    }
                    if (Trackers[i].FoundCount < 4)
                    {
                        Trackers.RemoveAt(i);
                    }
                }
            }

            // research every fifth frame.
            searchState = (searchState + 1) % 5;

        }

        /// <summary>
        /// Mark a location.
        /// </summary>
        /// <param name="b"></param>
        /// <param name="name"></param>
        /// <param name="t"></param>
        /// <param name="colorIndex"></param>
        protected void marklocation(Bitmap b, string name, Target t, Rectangle bounds, int colorIndex)
        {
            Color[] clrs = { Color.Red, Color.Green, Color.Blue, Color.Orange, Color.Magenta, Color.Cyan };
            Font f = Font;
            using (Graphics g = Graphics.FromImage(b))
            {
                using (Pen p = new Pen(Color.FromArgb(128, clrs[colorIndex % clrs.Length]), 4f))
                {
                    g.DrawRectangle(p, bounds);

                    if ((f == null) || (name == null))
                    {
                        // done.
                        return;
                    }

                    SizeF sft = g.MeasureString(name, f);
                    int sftw = 1 + (int)sft.Width;
                    int sfth = 1 + (int)sft.Height;

                    // Assume top left.
                    int distanceX = 20;
                    int distanceY = 10;
                    Point src = new Point(bounds.X + bounds.Width, bounds.Y + bounds.Height);
                    Point dst = new Point(src.X + distanceX, src.Y + distanceY);
                    Point tpos = new Point(dst.X, dst.Y);
                    if (bounds.X > b.Width / 2)
                    {
                        // Right
                        if (bounds.Y < b.Height / 2)
                        {
                            // Top right.
                            src = new Point(bounds.X, bounds.Y + bounds.Height);
                            dst = new Point(src.X - distanceX, src.Y + distanceY);
                            tpos = new Point(dst.X - sftw, dst.Y);
                        }
                        else
                        {
                            // Bottom right. 
                            src = new Point(bounds.X, bounds.Y);
                            dst = new Point(src.X - distanceX, src.Y - distanceY);
                            tpos = new Point(dst.X - sftw, dst.Y - sfth);
                        }
                    }
                    else
                    {
                        // Left, only do bottom left.
                        if (bounds.Y > b.Height / 2)
                        {
                            src = new Point(bounds.X + bounds.Width, bounds.Y);
                            dst = new Point(src.X + distanceX, src.Y - distanceY);
                            tpos = new Point(dst.X, dst.Y - sfth);
                        }
                    }
                    g.DrawLine(p, src, dst);
                    using (SolidBrush sb = new SolidBrush(Color.FromArgb(128, Color.Gray)))
                    {
                        g.FillRectangle(sb, tpos.X, tpos.Y, sftw, sfth);
                    }
                    using (SolidBrush sb = new SolidBrush(p.Color))
                    {
                        g.DrawString(name, f, sb, (float)tpos.X, (float)tpos.Y);
                    }
                }
            }
        }


        /// <summary>
        /// Object that is classified and tracked.
        /// </summary>
        protected class AssociativeTracker
        {
            /// <summary>
            /// Found in a row counter.
            /// </summary>
            public int FoundCount = 0;

            /// <summary>
            /// Found in a row counter.
            /// </summary>
            public bool FoundThisPass = false;

            /// <summary>
            /// Constructor.
            /// </summary>
            /// <param name="f"></param>
            /// <param name="targetIndex"></param>
            /// <param name="position"></param>
            public AssociativeTracker(Feature f, int targetIndex, Rectangle position)
            {
                Position = new PositionFilter();
                
                Position.Reset(position.X, position.Y);
                for (int i = 0; i < 10; i++)
                {
                    Position.Update(position);
                }
                TargetIndex = targetIndex;
                Feature = f;
                FoundCount = 0;
            }

            /// <summary>
            /// Default constructor.
            /// </summary>
            public AssociativeTracker()
            {
                Position = new PositionFilter();
                TargetIndex = 0;
                Feature = null;
                FoundCount = 0;
            }

            /// <summary>
            /// The feature.
            /// </summary>
            public Feature Feature { get; set; }

            /// <summary>
            /// The target index.
            /// </summary>
            public int TargetIndex { get; set; }

            /// <summary>
            /// Position.
            /// </summary>
            public PositionFilter Position { get; set; }

            /// <summary>
            /// Utility conversion.
            /// </summary>
            /// <param name="r"></param>
            /// <returns></returns>
            protected CvRect toCvRect(Rectangle r)
            {
                return new CvRect(r.X, r.Y, r.Width, r.Height);
            }

            /// <summary>
            /// First pass attempts to match within search window last target matched, return true on success.
            /// Only last target, any match, in window.
            /// </summary>
            /// <param name="img"></param>
            /// <param name="imgOut"></param>
            /// <returns></returns>
            public bool FirstPass(IplImage img, Bitmap imgOut)
            {
                return testTarget(img, imgOut, true, false, 1, TargetIndex);
            }

            /// <summary>
            /// First pass attempts to match within search window all targets.
            /// All targets, perfect match, in window.
            /// </summary> 
            /// <param name="img"></param>
            /// <param name="imgOut"></param>
            /// <returns></returns>
            public bool SecondPass(IplImage img, Bitmap imgOut)
            {
                for (int i = 0; i < Feature.Targets.Count; i++)
                {
                    if (i == TargetIndex) continue;
                    if (testTarget(img, imgOut, false, false, 1, i))
                    {
                        return true;
                    }
                }
                return false;
            }

            /// <summary>
            /// Third pass.
            /// </summary>
            /// <param name="img"></param>
            /// <param name="imgOut"></param>
            /// <returns></returns>
            public bool ThirdPass(IplImage img, Bitmap imgOut)
            {
                CvRect searchRect = new CvRect(0, 0, img.Width, img.Height);
                int bestTarget = 0;
                int bestX = 0;
                int bestY = 0;
                bool foundAny = false;
                float bestdistance = (img.Width * img.Width) + (img.Height * img.Height);
                float minimumBest = bestdistance / 2;
                for (int i = 0; i < Feature.Targets.Count; i++)
                {
                    List<Rectangle> rects = Feature.Targets[i].Find(
                            img,
                            searchRect,
                            Feature.CrossCorrelationThreshold,
                            10);
                    if (rects == null) return false;
                    if (rects.Count == 0) return false;

                    // Find the closest to our position aka associate.

                    for (int j = 0; j < rects.Count; j++)
                    {
                        int x = Position.X;
                        int y = Position.Y;
                        float dx = rects[j].X - x;
                        float dy = rects[j].Y - y;
                        float distance = (dx * dx) + (dy * dy); // skip sqrt.
                        if (distance < bestdistance)
                        {
                            bestdistance = distance;
                            bestX = x;
                            bestY = y;
                            bestTarget = i;
                            foundAny = true;
                        }
                    }
                }
                if (foundAny && (bestdistance < minimumBest))
                {
                    Position.Update(new Rectangle(bestX, bestY, Position.Width, Position.Height));
                    TargetIndex = bestTarget;
                    FoundCount = FoundCount >= 0 ? FoundCount + 1 : 1;
                    FoundThisPass = true;
                    return true;
                }
                return false;
            }

            /// <summary>
            /// Test target match.
            /// </summary>
            /// <param name="img"></param>
            /// <param name="imgOut"></param>
            /// <param name="perfect"></param>
            /// <param name="index"></param>
            /// <returns></returns>
            protected bool testTarget(IplImage img, Bitmap imgOut, bool perfect, bool wholeImage, int max, int index)
            {
                CvRect searchRect = new CvRect(0, 0, img.Width, img.Height);
                List<Rectangle> rects = Feature.Targets[index].Find(
                            img,
                            wholeImage ? searchRect : toCvRect(Position.SearchBounds),
                            perfect ? Feature.AutoCorrelationThreshold : Feature.CrossCorrelationThreshold,
                            max);
                if (rects == null) return false;
                if (rects.Count == 0) return false;
                Position.Update(rects[0]);
                TargetIndex = index;
                FoundThisPass = true;
                FoundCount = FoundCount >= 0 ? FoundCount + 1 : 1;
                Console.WriteLine(rects[0].X + " " + rects[0].Y + " " + Position.X + " " + Position.Y);
                return true;
            }
        }
    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

HoshiKata
Software Developer (Senior) KMC Systems
United States United States
Phil is a Principal Software developer focusing on weird yet practical algorithms that run the gamut of embedded and desktop (PID loops, Kalman filters, FFTs, client-server SOAP bindings, ASIC design, communication protocols, game engines, robotics).
 
In his personal life he is a part time mad scientist, full time dad, and studies small circle jujitsu, plays guitar and piano.

| Advertise | Privacy | Terms of Use | Mobile
Web03 | 2.8.141223.1 | Last Updated 23 Apr 2013
Article Copyright 2012 by HoshiKata
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid