Benchmark start-up and system performance for .Net, Mono, Java, C++ and their respective UI

// BenchMarkStartup.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include <windows.h>
#include <assert.h>
#include "statistics.h"

#include <psapi.h>
#pragma comment(lib, "Psapi.lib")
#include <atlstr.h>
#include "MainWindowFinder.h"

BOOL PrepareColdStart()
{
    // Get the list of process identifiers.
	if(!EmptyWorkingSet(GetCurrentProcess()))
	{
		printf( "EmptyWorkingSet() failed for BenchMarkStartup.exe, PID = %x\n",::GetCurrentProcessId() );
		return FALSE;
	}

    DWORD aProcesses[1024], cbNeeded, cProcesses;
    unsigned int i;

    if ( !EnumProcesses( aProcesses, sizeof(aProcesses), &cbNeeded ) )
        return FALSE;

    // Calculate how many process identifiers were returned.

    cProcesses = cbNeeded / sizeof(DWORD);

    // Print the memory usage for each process

    for ( i = 0; i < cProcesses; i++ )
	{
		if(aProcesses[i] == 0)
			continue;
		HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |PROCESS_SET_QUOTA,
								FALSE, aProcesses[i] );

		if (NULL == hProcess)
		{
			//printf( "Failed to Open Process ID: 0x%08x\n", aProcesses[i]);
			continue;
		}
		else
		{
			
			if(!EmptyWorkingSet(hProcess))
				printf( "EmptyWorkingSet failed for PID %x\n", aProcesses[i] );
			CloseHandle( hProcess );
		}

	}//for ends
	return TRUE;
}


__int64 currentWindowsFileTime()
{
  static const __int64 startEpoch2 = 0; // 1601/1/1
  FILETIME   ft;
  __int64 *pcrt;
#ifdef _DEBUG
 SYSTEMTIME st;
 st.wYear = 1601;
 st.wMonth = 1;
 st.wDay =1;
 st.wHour = 0;
 st.wMinute = 0;
 st.wSecond = 0;
 st.wMilliseconds = 0;
 BOOL bres = ::SystemTimeToFileTime(&st,&ft); 
  pcrt = reinterpret_cast<__int64 *>(&ft);
  assert(*pcrt == startEpoch2);
#endif
  GetSystemTimeAsFileTime(&ft);
  pcrt = reinterpret_cast<__int64 *>(&ft);
  return (*pcrt - startEpoch2); 
}

TCHAR strErr[200];
LPCTSTR GetErrorDescription(INT code)
{
	if(!code)
		return _T("Success");
	TCHAR* lpMsgBuf = NULL;
	FormatMessage( 
		FORMAT_MESSAGE_ALLOCATE_BUFFER | 
		FORMAT_MESSAGE_FROM_SYSTEM | 
		FORMAT_MESSAGE_IGNORE_INSERTS,
		NULL,
		code,
		0, // Default language
		(LPTSTR) &lpMsgBuf,
		0,
		NULL);

	if(lpMsgBuf)
	{
		//g_strErr = lpMsgBuf;
		_tcscpy_s(strErr,200, lpMsgBuf);
		LocalFree(lpMsgBuf);
	}
	else
		return _T("Unknown Error");
	return strErr;
}

#ifdef _DEBUG
const int COUNT = 3;
#else
const int COUNT = 9;
#endif

int strtupTimes[COUNT];
int kernelTimes[COUNT];
int preCreationTimes[COUNT];
int userTimes[COUNT];

HWND MinimizeUIApp(const PROCESS_INFORMATION pi)
{
		__int64 start;
		SYSTEMTIME st;
		::GetSystemTime(&st);
		SystemTimeToFileTime(&st, reinterpret_cast<LPFILETIME>(&start));
		HWND hwnd = NULL;
		DWORD dwerr = 0;

		do
		{

			__int64 crttime;
			SYSTEMTIME ct;
			::GetSystemTime(&ct);
			SystemTimeToFileTime(&ct, reinterpret_cast<LPFILETIME>(&crttime));
			if(crttime - start > 90000000)//9 second timeout
			{
				_tprintf( _T("FindMainWindow timed out after %I64d seconds\n"), (crttime - start)/10000000 );
				break;
			}
			::SetLastError(0);
			// Wait up to 20 seconds or until the child process is ready.
			dwerr = ::WaitForInputIdle( pi.hProcess, 20000 );
			if(dwerr != 0)
			{
				::Sleep(100);
			}

			MainWindowFinder mwf(pi.dwProcessId);
			hwnd = mwf.FindMainWindow();
			if(!hwnd)
				::Sleep(500);
		}
		while(!hwnd);
		if(hwnd)
		{
			::SendMessage(hwnd,WM_SYSCOMMAND,SC_MINIMIZE,NULL);
		}
		else
		{
			dwerr = GetLastError();
			_tprintf( _T("FindMainWindow failed with error code %d\n"), dwerr );
			// Close process and thread handles. 
			TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
		}
		return hwnd;
}

BOOL CloseUIApp(const PROCESS_INFORMATION pi)
{
		__int64 start;
		SYSTEMTIME st;
		::GetSystemTime(&st);
		SystemTimeToFileTime(&st, reinterpret_cast<LPFILETIME>(&start));
		HWND hwnd = NULL;
		DWORD dwerr = 0;

		do
		{

			__int64 crttime;
			SYSTEMTIME ct;
			::GetSystemTime(&ct);
			SystemTimeToFileTime(&ct, reinterpret_cast<LPFILETIME>(&crttime));
			if(crttime - start > 90000000)//9 second timeout
			{
				_tprintf( _T("FindMainWindow timed out after %I64d seconds\n"), (crttime - start)/10000000 );
				break;
			}
			::SetLastError(0);
			// Wait up to 20 seconds or until the child process is ready.
			dwerr = ::WaitForInputIdle( pi.hProcess, 20000 );
			if(dwerr != 0)
			{
				::Sleep(100);
			}

			MainWindowFinder mwf(pi.dwProcessId);
			hwnd = mwf.FindMainWindow();
			if(!hwnd)
				::Sleep(500);
		}
		while(!hwnd);
		if(hwnd)
		{
			::SendMessage(hwnd,WM_CLOSE,NULL,NULL);
		}
		else
		{
			dwerr = GetLastError();
			_tprintf( _T("FindMainWindow failed with error code %d\n"), dwerr );
			// Close process and thread handles. 
			TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
			return FALSE;
		}
		return TRUE;
}

DWORD BenchMarkTimes( LPCTSTR szcProg)
{
	ZeroMemory( strtupTimes, sizeof(strtupTimes) );
	ZeroMemory( kernelTimes, sizeof(kernelTimes) );
	ZeroMemory( preCreationTimes, sizeof(preCreationTimes) );
	ZeroMemory( userTimes, sizeof(userTimes) );
	BOOL res = TRUE;
	TCHAR cmd[100];
	int i,result = 0;
	DWORD dwerr = 0;
	PrepareColdStart();
	::Sleep(3000);//3 seconds delay


	for(i = 0; i <= COUNT && res; i++)
	{
		STARTUPINFO si;
		PROCESS_INFORMATION pi;
		ZeroMemory( &si, sizeof(si) );
		si.cb = sizeof(si);
		ZeroMemory( &pi, sizeof(pi) );
		::SetLastError(0);
		__int64 wft = 0;
		if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
		{ //java -client -cp .\.. JavaPerf.StartupTest
			wft = currentWindowsFileTime();
			//will use javaw instead of java
			_stprintf_s(cmd,100,_T("javaw.exe -client -cp .\\.. %s \"%I64d\""), szcProg,wft);
		}
		else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
		{ //mono ..\monoperf\mono26perf.exe
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("mono %s \"%I64d\""), szcProg,wft);
		}
		else
		{
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("%s \"%I64d\""), szcProg,wft);		 
		}

		// Start the child process. 
		if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
		{
			dwerr = GetLastError();
			_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
			return dwerr;
		}
		
		if(!CloseUIApp(pi))
			break;

		// Wait up to 20 seconds or until the child process exits.
		dwerr = WaitForSingleObject( pi.hProcess, 20000 );
		if(dwerr != WAIT_OBJECT_0)
		{
			dwerr = GetLastError();
			_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
			// Close process and thread handles. 
			//TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
			break;
		}

		res = GetExitCodeProcess(pi.hProcess,(LPDWORD)&result);
		FILETIME CreationTime,ExitTime,KernelTime,UserTime;
		if(GetProcessTimes(pi.hProcess,&CreationTime,&ExitTime,&KernelTime,&UserTime))
		{
			__int64 *pKT,*pUT, *pCT;
			pKT = reinterpret_cast<__int64 *>(&KernelTime);
			pUT = reinterpret_cast<__int64 *>(&UserTime);
			pCT = reinterpret_cast<__int64 *>(&CreationTime);
			if(i == 0)
			{
				_tprintf( _T("cold start times:\nStartupTime %d ms"), result);
				_tprintf( _T(", PreCreationTime: %u ms"), ((*pCT)- wft)/ 10000); 
				_tprintf( _T(", KernelTime: %u ms"), (*pKT) / 10000);
				_tprintf( _T(", UserTime: %u ms\n"), (*pUT) / 10000); 
				_tprintf( _T("Waiting for statistics for %d warm samples"), COUNT); 
			}
			else
			{
				_tprintf( _T("."));
				kernelTimes[i-1] = (int)((*pKT) / 10000);
				preCreationTimes[i-1] = (int)((*pCT)- wft)/ 10000; 
				userTimes[i-1] = (int)((*pUT) / 10000);
				strtupTimes[i-1] = result;	
			}
		}
		else
		{
			printf( "GetProcessTimes failed for %p",  pi.hProcess ); 
		}
		// Close process and thread handles. 
		CloseHandle( pi.hProcess );
		CloseHandle( pi.hThread );
		if((int)result < 0)
		{
			_tprintf( _T("%s failed with code %d: %s\n"),cmd, result,GetErrorDescription(result) );
			return result;
		}
		::Sleep(1000); //1s delay

	}//for ends
	if(i <= COUNT )
	{
		result = GetLastError();
		_tprintf( _T("\nThere was an error while running '%s', last error code = %d: %s\n"),cmd,result,GetErrorDescription(result));
		return result;
	}

	double median, mean, stddev;
	if(CalculateStatistics(&strtupTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("\nStartupTime: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&preCreationTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("PreCreation: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&kernelTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("KernelTime : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&userTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("UserTime   : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}

	return GetLastError();
}

DWORD BenchMarkMemory( LPCTSTR szcProg)
{
	TCHAR cmd[100];
	STARTUPINFO si;
	PROCESS_INFORMATION pi;
	ZeroMemory( &si, sizeof(si) );
	si.cb = sizeof(si);
	ZeroMemory( &pi, sizeof(pi) );
	DWORD dwerr;
	if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("javaw -client -cp .\\.. %s"), szcProg);
	}
	else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("mono %s"), szcProg);
	}
	else
	{
			_stprintf_s(cmd,100,_T("%s"), szcProg);		 
	}
	::SetLastError(0);
	// Start the child process. 
	if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
	{
		dwerr = GetLastError();
		_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
		return dwerr;
	}

	//wait to show up
	::Sleep(3000);

    PROCESS_MEMORY_COUNTERS_EX pmc;
    if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
    {
		printf( "Normal size->PrivateUsage: %lu KB,",  pmc.PrivateUsage/1024 );
        printf( " Current WorkingSet: %lu KB\n", pmc.WorkingSetSize/1024 );
    }
	else
	{
		printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 

	}

	HWND hwnd = MinimizeUIApp(pi);
	//wait to minimize
	::Sleep(2000);

    if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
    {
		printf( "Minimized->  PrivateUsage: %lu KB,",  pmc.PrivateUsage/1024 );
        printf( " Current WorkingSet: %lu KB\n", pmc.WorkingSetSize/1024 );
    }
	else
		printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 
	

	if(!EmptyWorkingSet(pi.hProcess))
		printf( "EmptyWorkingSet failed for %x\n", pi.dwProcessId );
	else
	{
		ZeroMemory(&pmc, sizeof(pmc));
		if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
		{
		
			printf( "Minimum WorkingSet:        %lu KB,", pmc.WorkingSetSize/1024 );
			printf( " PeakWorkingSet:     %lu KB\n",  pmc.PeakWorkingSetSize/1024 );  
		}
		else
			printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 
	}

  //	if(!SetProcessWorkingSetSize(pi.hProcess, -1, -1))
		//printf( "SetProcessWorkingSetSize failed\n" );
	if(hwnd)
		::SendMessage(hwnd,WM_CLOSE,NULL,NULL);

	// Wait up to 20 seconds or until the child process exits.
	dwerr = WaitForSingleObject( pi.hProcess, 20000 );
	if(dwerr != WAIT_OBJECT_0)
	{
		dwerr = GetLastError();
		_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
	}
	// Close process and thread handles. 
	CloseHandle( pi.hProcess );
	CloseHandle( pi.hThread );
	return GetLastError();
}
void _tmain( int argc, TCHAR *argv[] )
{

    if( argc < 2 )
    {
        _tprintf( _T("Usage: %s [test1.exe test2.exe java_class....]\n"), argv[0]);
        return;
    }

	for(int i = 1; i < argc; i++)
	{
		_tprintf( _T("Benchmarking %s\n"), argv[i] );
		if(!BenchMarkTimes(argv[i]))
			BenchMarkMemory(argv[i]);
		_tprintf( _T("________________________________________________________________________________\n"));
	}
}