Advanced Unit Testing, Part IV - Fixture Setup/Teardown, Test Repetition And Performance Tests

using System;
using System.Diagnostics;
using System.Threading;

using KALib;

namespace UTCore
{
	public abstract class TestFixtureRunner
	{
		protected TestFixture tf;

		public abstract void RunTests();
		public abstract bool ExceptionConfirmed(Exception e, TestAttribute ta);

		protected TestFixtureRunner(TestFixture tf)
		{
			this.tf=tf;
		}

		protected TestAttribute.TestState RunTest(object instance, TestAttribute ta)
		{
			if (!ta.IgnoreTest)
			{
				SortableList runTimeList=new SortableList();
				for (int i=0; i<ta.TestMethod.RepeatCount; i++)
				{
					ta.TestMethod.CurrentRep=i;
					try
					{
						ta.Result="";
						tf.TestSetUp.Invoke(instance);
						Thread.Sleep(0);	// let other threads execute
						ta.Invoke(instance);
						// If we get here, the test did not throw an exception.
						// Was it supposed too?
						if (ta.ExpectedExceptionType != null)
						{
							Trace.WriteLine("***Fail***: "+ta.TestMethod.ToString()+" Expected exception not encountered");
							ta.State=TestAttribute.TestState.Fail;
							ta.Result="Expected exception "+ta.ExpectedExceptionType.ToString()+" not encountered.";
						}

							// did we meet the min. operations per second requirement?
						else if ( (ta.TestMethod.MinOPS > 0) && (1.0/ta.dTime < ta.TestMethod.MinOPS) )
						{
							Trace.WriteLine("***Fail***: "+ta.TestMethod.ToString()+" Min. operations/sec not met.");
							ta.State=TestAttribute.TestState.Fail;
							ta.Result="Min. operations/sec not met.";
						}
							// did we meet the max. memory allowed?
						else if ( (ta.TestMethod.MaxK > 0) && (ta.TestMethod.MemoryUsed > ta.TestMethod.MaxK*1000) )
						{
							Trace.WriteLine("***Fail***: "+ta.TestMethod.ToString()+" Max. memory exceeded");
							ta.State=TestAttribute.TestState.Fail;
							ta.Result="Max. memory utilization exceeded.";
						}

						else
						{
							Trace.WriteLine("***Pass***: "+ta.TestMethod.ToString());
							ta.State=TestAttribute.TestState.Pass;
						}
					}

					catch(UnitTest.AssertionException e)
					{
						Trace.WriteLine("***Fail***: "+ta.TestMethod.ToString()+" Exception="+e.Message);
						ta.State=TestAttribute.TestState.Fail;
						ta.Result="Assertion failed: "+e.Message;
					}

					catch(Exception e)
					{
						if (ExceptionConfirmed(e, ta))
						{
							Trace.WriteLine("***Pass***: "+ta.TestMethod.ToString()+" Exception="+e.Message);
							ta.State=TestAttribute.TestState.Pass;
						}
						else
						{
							Trace.WriteLine("***Fail***: "+ta.TestMethod.ToString()+" Exception="+e.Message);
							ta.State=TestAttribute.TestState.Fail;
							ta.Result="Exception occurred: "+e.Message;
						}
					}
					finally
					{
						runTimeList.Add(ta.dTime);
						tf.TestTearDown.Invoke(instance);
						if (ta.TestMethod.RepeatCount > 1)
						{
							// only fire this event when we're really running a test repeatedly
							tf.IterationEvent(ta);
						}
					}

					// if all the repetitions succeeded, then take the average execution time, ignoring the
					// the best and worst times.  (This also ignores samples under 3).
					double total=0;
					for (int j=1; j<runTimeList.Count-1; j++)
					{
						total+=(double)runTimeList[j];
					}
					ta.dTime=(total==0.0 ? ta.dTime : ta.dTime)/runTimeList.Count;
					runTimeList.Clear();

					if (ta.TestMethod.RepeatDelay > 0)
					{
						Thread.Sleep(ta.TestMethod.RepeatDelay);
					}

				}
			}
			else
			{
				Trace.WriteLine("***Ignore***: "+ta.TestMethod.ToString());
				ta.Result=ta.IgnoreReason;
				ta.State=TestAttribute.TestState.Ignore;
			}
			tf.NotificationEvent(ta);
			return ta.State;
		}
	}
}