Click here to Skip to main content
Click here to Skip to main content
 
Add your own
alternative version
Go to top

Image Tracking and Computer Vision Using Fourier Image Correlation

, 23 Apr 2013
How to teach a program to recognize something within a video stream.
OpenCVDemo3-noexe.zip
OpenCVDemo
Description.odt
OpenCvSharp-2.3.1-x86-20111030
XmlDoc-English
XmlDoc-Japanese
TrackingEyes.jpg
WindowsFormsApplication1
Properties
Settings.settings
OpenCVDemo3.zip
Description.odt
OpenCv-2.3.1-x86
opencv_calib3d231.dll
opencv_contrib231.dll
opencv_core231.dll
opencv_features2d231.dll
opencv_ffmpeg.dll
opencv_flann231.dll
opencv_gpu231.dll
opencv_highgui231.dll
opencv_imgproc231.dll
opencv_legacy231.dll
opencv_ml231.dll
opencv_objdetect231.dll
opencv_ts231.dll
opencv_video231.dll
tbb.dll
OpenCvSharp.Blob.dll
OpenCvSharp.CPlusPlus.dll
OpenCvSharp.DebuggerVisualizers.dll
OpenCvSharp.dll
OpenCvSharp.Extensions.dll
OpenCvSharp.MachineLearning.dll
OpenCvSharp.UserInterface.dll
OpenCvSharpExtern.dll
TrackingEyes.jpg
Settings.settings
///////////////////////////////////////////////////////////////////////////////
//
//  Tracker.cs
//
//  By Philip R. Braica (HoshiKata@aol.com, VeryMadSci@gmail.com)
//
//  Distributed under the The Code Project Open License (CPOL)
//  http://www.codeproject.com/info/cpol10.aspx
///////////////////////////////////////////////////////////////////////////////

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Drawing;
using System.Data;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using OpenCvSharp;

namespace OpenCVDemo
{
    /// <summary>
    /// Class for tracking.
    /// </summary>
    public class Tracker
    { 
        /// <summary>
        /// Target list.
        /// </summary>
        protected List<Target> m_targets = new List<Target>();

        /// <summary>
        /// Marker list.
        /// </summary>
        protected ImageMarkerList m_markerList = new ImageMarkerList();

        /// <summary>
        /// Reset learning.
        /// </summary>
        public void Reset()
        {
            for (int i = 0; i < m_targets.Count; i++) m_targets[i].Dispose();
            m_targets.Clear();
        }

        /// <summary>
        /// The font.
        /// </summary>
        public Font Font { get; set; }

        /// <summary>
        /// Set targets.
        /// </summary>
        /// <param name="targets"></param>
        public void SetTargets(List<Target> targets)
        {
            m_targets.Clear();
            m_targets.AddRange(targets);
        }

        /// <summary>
        /// Track an object in frames based on target array.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="imgOut"></param>
        /// <param name="r"></param>
        public void Track(IplImage img, Bitmap imgOut, Rectangle r)
        {
            // Not the initial case.
            // Search around last detection.
            int best = 0;
            float bestScore = 0;
            CvRect searchRect = makeSearchRectangle(img.Width, img.Height, 50);
            for (int i = 0; i < m_targets.Count; i++)
            {
                float score = m_targets[i].Correlate(img, searchRect);
                if (score > AutoCorrelationThreshold)
                {
                    // We found it, no need to do better.
                    markLocation(imgOut, m_targets[i].LastRect);
                    return;
                }
                if (score > bestScore)
                {
                    best = i; bestScore = score;
                }
            }

            // If needed expand the search.
            if (bestScore < CrossCorrelationThreshold)
            {
                searchRect = new CvRect(0, 0, img.Width, img.Height);
                for (int i = 0; i < m_targets.Count; i++)
                {
                    float score = m_targets[i].Correlate(img, searchRect);
                    if (score > AutoCorrelationThreshold)
                    {
                        // We found it, no need to do better.
                        markLocation(imgOut, m_targets[i].LastRect);                        
                        return;
                    }
                    if (score > bestScore)
                    {
                        best = i; bestScore = score;
                    }
                }
            }

            if (bestScore > CrossCorrelationThreshold)
            {
                markLocation(imgOut, m_targets[best].LastRect);
            }
        }

        /// <summary>
        /// Save feature.
        /// </summary>
        /// <param name="fileName"></param>
        public void SaveFeature(string fileName, string featureName)
        {
            int cnt = m_targets.Count;
            //serialize and persist it to it's file
            try
            {
                Feature TC = new Feature(
                    featureName,
                    m_targets, 
                    AutoCorrelationThreshold, 
                    LearningCorrelationThreshold, 
                    CrossCorrelationThreshold);
                TC.Save(fileName);
            }
            catch (Exception)
            {
                // Fail silently for now.
            }
        }

        /// <summary>
        /// Learn, make a target array.
        /// </summary>
        /// <param name="img"></param>
        /// <param name="imgOut"></param>
        /// <param name="r"></param>
        public void Learn(IplImage img, Bitmap imgOut, Rectangle r, bool doStats)
        {
            m_markerList.Clear();

            // Handle the initial case if m_targets.Count == 0.
            if (m_targets.Count == 0)
            {
                markLocation(imgOut, r);
                m_targets.Add(new Target(img, r));
                return;
            }
            int best = 0;
            float bestScore = 0;
            if (doStats)
            {
                CvRect searchRect = new CvRect(0, 0, img.Width, img.Height);
                for (int i = 0; i < m_targets.Count; i++)
                {
                    float score = m_targets[i].Correlate(img, searchRect);
                    if (m_targets[i].Enabled)
                    {
                        if (score > bestScore)
                        {
                            best = i; bestScore = score;
                        }
                    }
                    if (score >= CrossCorrelationThreshold)
                    {
                        m_markerList.Add((score * 100).ToString("F1"), m_targets[i].LastRect);
                    }
                }
            }
            else
            {
                // Not the initial case.
                // Search around last detection.
                CvRect searchRect = makeSearchRectangle(img.Width, img.Height, 50);
                for (int i = 0; i < m_targets.Count; i++)
                {
                    if (m_targets[i].Enabled == false)
                    {
                        m_targets[i].Scores.Add(0);
                        continue;
                    }
                    float score = m_targets[i].Correlate(img, searchRect);
                    if (score > AutoCorrelationThreshold)
                    {
                        // We found it, no need to do better.
                        markLocation(imgOut, m_targets[i].LastRect);
                        for (int j = i; j < m_targets.Count; j++)
                        {
                            m_targets[i].Scores.Add(0);
                        }
                        m_markerList.Add((score * 100).ToString("F1"), m_targets[i].LastRect);
                        return;
                    }
                    if (score > bestScore)
                    {
                        best = i; bestScore = score;
                    }
                    if (score >= CrossCorrelationThreshold)
                    {
                        m_markerList.Add((score * 100).ToString("F1"), m_targets[i].LastRect);
                    }
                }

                // If needed expand the search.
                if (bestScore < CrossCorrelationThreshold)
                {
                    searchRect = new CvRect(0, 0, img.Width, img.Height);
                    for (int i = 0; i < m_targets.Count; i++)
                    {
                        if (m_targets[i].Enabled == false)
                        {
                            m_targets[i].Scores.Add(0);
                            continue;
                        }
                        float score = m_targets[i].Correlate(img, searchRect);
                        if (score > AutoCorrelationThreshold)
                        {
                            // We found it, no need to do better.
                            markLocation(imgOut, m_targets[i].LastRect);
                            for (int j = i; j < m_targets.Count; j++)
                            {
                                m_targets[i].Scores.Add(0);
                            }
                            m_markerList.Add((score * 100).ToString("F1"), m_targets[i].LastRect);
                            return;
                        }
                        if (score > bestScore)
                        {
                            best = i; bestScore = score;
                        }
                        if (score >= CrossCorrelationThreshold)
                        {
                            m_markerList.Add((score * 100).ToString("F1"), m_targets[i].LastRect);
                        }
                    }
                }
            }

            if (bestScore > CrossCorrelationThreshold)
            {
                markLocation(imgOut, m_targets[best].LastRect);
                if (bestScore <= AutoCorrelationThreshold)
                {
                    m_targets.Add(new Target(img, m_targets[best].LastRect));

                    // Omitting first and just added, if > Replace, remove it.
                    if ((best != 0) && (bestScore > LearningCorrelationThreshold))
                    {
                        m_targets[best].Enabled = false;
                    }
                }
            }

            if (doStats)
            {
                for (int i = 0; i < m_targets.Count; i++)
                {
                    m_targets[i].CrossCorrelate(CurrentLocation);
                }
            }
            m_markerList.Render(imgOut, Font);
        }

        /// <summary>
        /// Make a log report.
        /// </summary>
        /// <returns></returns>
        public string MakeLogReport()
        {
            StringBuilder sb = new StringBuilder();
            int max = 0;
            for (int i = 0; i < m_targets.Count; i++)
            {
                max = m_targets[i].Scores.Count > max ? m_targets[i].Scores.Count : max;
            }
            if (max > 0)
            {
                sb.Append("Correlations\r\n\r\n");
                for (int row = 0; row < max; row++)
                {
                    for (int i = 0; i < m_targets.Count; i++)
                    {
                        int delta = row + m_targets[i].Scores.Count - max;
                        if (delta >= 0)
                        {
                            sb.Append(m_targets[i].Scores[delta]);
                           
                        }
                        
                        if (i != m_targets.Count - 1) sb.Append(", ");
                    }
                    sb.Append("\r\n");
                }
                sb.Append("\r\nCross correlations\r\n\r\n");
                for (int row = 0; row < max; row++)
                {
                    for (int i = 0; i < m_targets.Count; i++)
                    {
                        int delta = row + m_targets[i].Scores.Count - max;
                        if (delta >= 0)
                        {
                            sb.Append(m_targets[i].CrossScores[delta]);
                        }

                        if (i != m_targets.Count - 1) sb.Append(", ");
                    }
                    sb.Append("\r\n");
                }
            }

            return sb.ToString();
        }

        /// <summary>
        /// Threshold to call off match attempt we have an ideal.
        /// </summary>
        public float AutoCorrelationThreshold = 0.95f;

        /// <summary>
        /// Good enough to declare a match threshold.
        /// </summary>
        public float CrossCorrelationThreshold = 0.75f;

        /// <summary>
        /// Good enough to replace threshold.
        /// </summary>
        public float LearningCorrelationThreshold = 0.86f;


        public Rectangle CurrentLocation = new Rectangle();

        /// <summary>
        /// Mark the location.
        /// </summary>
        /// <param name="b"></param>
        /// <param name="r"></param>
        protected void markLocation(Bitmap b, Rectangle r)
        {
            CurrentLocation = r;
            using (Graphics g = Graphics.FromImage(b))
            {
                using (Pen p = new Pen(Color.FromArgb(128, Color.Orange), 4f))
                {
                    g.DrawRectangle(p, r);
                }
            }
        }

        /// <summary>
        /// Make a search rectangle
        /// </summary>
        /// <param name="maxWidth"></param>
        /// <param name="maxHeight"></param>
        /// <param name="expansion"></param>
        /// <returns></returns>
        protected CvRect makeSearchRectangle(int maxWidth, int maxHeight, int expansion)
        {
            int x = CurrentLocation.X - expansion;
            int y = CurrentLocation.Y - expansion;
            int w = CurrentLocation.Width + expansion + expansion;
            int h = CurrentLocation.Height + expansion + expansion;
            x = x < 0 ? 0 : x;
            y = y < 0 ? 0 : y;
            w = x + w < maxWidth ? w : maxWidth - x - 1;
            h = y + h < maxHeight ? h : maxHeight - y - 1;

            return new CvRect(x, y, w, h);
        }
    }

}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

HoshiKata
Software Developer (Senior) KMC Systems
United States United States
Phil is a Principal Software developer focusing on weird yet practical algorithms that run the gamut of embedded and desktop (PID loops, Kalman filters, FFTs, client-server SOAP bindings, ASIC design, communication protocols, game engines, robotics).
 
In his personal life he is a part time mad scientist, full time dad, and studies small circle jujitsu, plays guitar and piano.

| Advertise | Privacy | Mobile
Web01 | 2.8.140926.1 | Last Updated 23 Apr 2013
Article Copyright 2012 by HoshiKata
Everything else Copyright © CodeProject, 1999-2014
Terms of Service
Layout: fixed | fluid