Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Benchmark start-up and system performance for .Net, Mono, Java, C++ and their respective UI

, 2 Sep 2010 CPOL
What is the start-up and system performance overhead for .Net, Mono, Java versus C++ and Forms, WPF, Swing versus MFC
BenchMarkStartup_src.zip
BenchMarkStartup
BenchMarkStartup
BenchMarkStartup.vcxproj.user
BenchMarkStartupUI
BenchMarkStartupUI.vcxproj.user
CPPMFCPerf
CPPMFCPerf.vcxproj.user
res
CPPMFCPerf.ico
Debug
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
DotNet11Perf
App.ico
DotNet11UIPerf
App.ico
DotNet20Perf
Properties
DotNet20UIPerf
Properties
Settings.settings
DotNet35Perf
Properties
DotNet35UIPerf
Properties
Settings.settings
DotNet35WPFPerf
Properties
Settings.settings
DotNet40Perf
Properties
DotNet40UIPerf
Properties
Settings.settings
DotNet40WPFPerf
Properties
Settings.settings
JavaPerf
StartupTest.class
SwingTest.class
MonoPerf
Mono26Perf.exe
Mono26UIPerf.exe
NativeCPPPerf
Release
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
NativeCPPPerf.exe
Release.zip
MonoPerf
Mono26Perf.exe
Mono26UIPerf.exe
Release
BenchMarkStartup.exe
BenchMarkStartupUI.exe
CPPMFCPerf.exe
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet20Perf.exe
DotNet20UIPerf.exe
DotNet35Perf.exe
DotNet35UIPerf.exe
DotNet35WPFPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
mfc100u.dll
msvcm90.dll
msvcp100.dll
msvcp90.dll
msvcr100.dll
msvcr90.dll
NativeCPPPerf.exe
JavaPerf
StartupTest.class
SwingTest.class
// BenchMarkStartup.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include <windows.h>
#include <assert.h>
#include "statistics.h"

#include <psapi.h>
#pragma comment(lib, "Psapi.lib")
#include <atlstr.h>
#include "MainWindowFinder.h"

BOOL PrepareColdStart()
{
    // Get the list of process identifiers.
	if(!EmptyWorkingSet(GetCurrentProcess()))
	{
		printf( "EmptyWorkingSet() failed for BenchMarkStartup.exe, PID = %x\n",::GetCurrentProcessId() );
		return FALSE;
	}

    DWORD aProcesses[1024], cbNeeded, cProcesses;
    unsigned int i;

    if ( !EnumProcesses( aProcesses, sizeof(aProcesses), &cbNeeded ) )
        return FALSE;

    // Calculate how many process identifiers were returned.

    cProcesses = cbNeeded / sizeof(DWORD);

    // Print the memory usage for each process

    for ( i = 0; i < cProcesses; i++ )
	{
		if(aProcesses[i] == 0)
			continue;
		HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |PROCESS_SET_QUOTA,
								FALSE, aProcesses[i] );

		if (NULL == hProcess)
		{
			//printf( "Failed to Open Process ID: 0x%08x\n", aProcesses[i]);
			continue;
		}
		else
		{
			
			if(!EmptyWorkingSet(hProcess))
				printf( "EmptyWorkingSet failed for PID %x\n", aProcesses[i] );
			CloseHandle( hProcess );
		}

	}//for ends
	return TRUE;
}


__int64 currentWindowsFileTime()
{
  static const __int64 startEpoch2 = 0; // 1601/1/1
  FILETIME   ft;
  __int64 *pcrt;
#ifdef _DEBUG
 SYSTEMTIME st;
 st.wYear = 1601;
 st.wMonth = 1;
 st.wDay =1;
 st.wHour = 0;
 st.wMinute = 0;
 st.wSecond = 0;
 st.wMilliseconds = 0;
 BOOL bres = ::SystemTimeToFileTime(&st,&ft); 
  pcrt = reinterpret_cast<__int64 *>(&ft);
  assert(*pcrt == startEpoch2);
#endif
  GetSystemTimeAsFileTime(&ft);
  pcrt = reinterpret_cast<__int64 *>(&ft);
  return (*pcrt - startEpoch2); 
}

TCHAR strErr[200];
LPCTSTR GetErrorDescription(INT code)
{
	if(!code)
		return _T("Success");
	TCHAR* lpMsgBuf = NULL;
	FormatMessage( 
		FORMAT_MESSAGE_ALLOCATE_BUFFER | 
		FORMAT_MESSAGE_FROM_SYSTEM | 
		FORMAT_MESSAGE_IGNORE_INSERTS,
		NULL,
		code,
		0, // Default language
		(LPTSTR) &lpMsgBuf,
		0,
		NULL);

	if(lpMsgBuf)
	{
		//g_strErr = lpMsgBuf;
		_tcscpy_s(strErr,200, lpMsgBuf);
		LocalFree(lpMsgBuf);
	}
	else
		return _T("Unknown Error");
	return strErr;
}

#ifdef _DEBUG
const int COUNT = 3;
#else
const int COUNT = 9;
#endif

int strtupTimes[COUNT];
int kernelTimes[COUNT];
int preCreationTimes[COUNT];
int userTimes[COUNT];

HWND MinimizeUIApp(const PROCESS_INFORMATION pi)
{
		__int64 start;
		SYSTEMTIME st;
		::GetSystemTime(&st);
		SystemTimeToFileTime(&st, reinterpret_cast<LPFILETIME>(&start));
		HWND hwnd = NULL;
		DWORD dwerr = 0;

		do
		{

			__int64 crttime;
			SYSTEMTIME ct;
			::GetSystemTime(&ct);
			SystemTimeToFileTime(&ct, reinterpret_cast<LPFILETIME>(&crttime));
			if(crttime - start > 90000000)//9 second timeout
			{
				_tprintf( _T("FindMainWindow timed out after %I64d seconds\n"), (crttime - start)/10000000 );
				break;
			}
			::SetLastError(0);
			// Wait up to 20 seconds or until the child process is ready.
			dwerr = ::WaitForInputIdle( pi.hProcess, 20000 );
			if(dwerr != 0)
			{
				::Sleep(100);
			}

			MainWindowFinder mwf(pi.dwProcessId);
			hwnd = mwf.FindMainWindow();
			if(!hwnd)
				::Sleep(500);
		}
		while(!hwnd);
		if(hwnd)
		{
			::SendMessage(hwnd,WM_SYSCOMMAND,SC_MINIMIZE,NULL);
		}
		else
		{
			dwerr = GetLastError();
			_tprintf( _T("FindMainWindow failed with error code %d\n"), dwerr );
			// Close process and thread handles. 
			TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
		}
		return hwnd;
}

BOOL CloseUIApp(const PROCESS_INFORMATION pi)
{
		__int64 start;
		SYSTEMTIME st;
		::GetSystemTime(&st);
		SystemTimeToFileTime(&st, reinterpret_cast<LPFILETIME>(&start));
		HWND hwnd = NULL;
		DWORD dwerr = 0;

		do
		{

			__int64 crttime;
			SYSTEMTIME ct;
			::GetSystemTime(&ct);
			SystemTimeToFileTime(&ct, reinterpret_cast<LPFILETIME>(&crttime));
			if(crttime - start > 90000000)//9 second timeout
			{
				_tprintf( _T("FindMainWindow timed out after %I64d seconds\n"), (crttime - start)/10000000 );
				break;
			}
			::SetLastError(0);
			// Wait up to 20 seconds or until the child process is ready.
			dwerr = ::WaitForInputIdle( pi.hProcess, 20000 );
			if(dwerr != 0)
			{
				::Sleep(100);
			}

			MainWindowFinder mwf(pi.dwProcessId);
			hwnd = mwf.FindMainWindow();
			if(!hwnd)
				::Sleep(500);
		}
		while(!hwnd);
		if(hwnd)
		{
			::SendMessage(hwnd,WM_CLOSE,NULL,NULL);
		}
		else
		{
			dwerr = GetLastError();
			_tprintf( _T("FindMainWindow failed with error code %d\n"), dwerr );
			// Close process and thread handles. 
			TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
			return FALSE;
		}
		return TRUE;
}

DWORD BenchMarkTimes( LPCTSTR szcProg)
{
	ZeroMemory( strtupTimes, sizeof(strtupTimes) );
	ZeroMemory( kernelTimes, sizeof(kernelTimes) );
	ZeroMemory( preCreationTimes, sizeof(preCreationTimes) );
	ZeroMemory( userTimes, sizeof(userTimes) );
	BOOL res = TRUE;
	TCHAR cmd[100];
	int i,result = 0;
	DWORD dwerr = 0;
	PrepareColdStart();
	::Sleep(3000);//3 seconds delay


	for(i = 0; i <= COUNT && res; i++)
	{
		STARTUPINFO si;
		PROCESS_INFORMATION pi;
		ZeroMemory( &si, sizeof(si) );
		si.cb = sizeof(si);
		ZeroMemory( &pi, sizeof(pi) );
		::SetLastError(0);
		__int64 wft = 0;
		if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
		{ //java -client -cp .\.. JavaPerf.StartupTest
			wft = currentWindowsFileTime();
			//will use javaw instead of java
			_stprintf_s(cmd,100,_T("javaw.exe -client -cp .\\.. %s \"%I64d\""), szcProg,wft);
		}
		else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
		{ //mono ..\monoperf\mono26perf.exe
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("mono %s \"%I64d\""), szcProg,wft);
		}
		else
		{
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("%s \"%I64d\""), szcProg,wft);		 
		}

		// Start the child process. 
		if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
		{
			dwerr = GetLastError();
			_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
			return dwerr;
		}
		
		if(!CloseUIApp(pi))
			break;

		// Wait up to 20 seconds or until the child process exits.
		dwerr = WaitForSingleObject( pi.hProcess, 20000 );
		if(dwerr != WAIT_OBJECT_0)
		{
			dwerr = GetLastError();
			_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
			// Close process and thread handles. 
			//TerminateProcess( pi.hProcess, -1 );
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
			break;
		}

		res = GetExitCodeProcess(pi.hProcess,(LPDWORD)&result);
		FILETIME CreationTime,ExitTime,KernelTime,UserTime;
		if(GetProcessTimes(pi.hProcess,&CreationTime,&ExitTime,&KernelTime,&UserTime))
		{
			__int64 *pKT,*pUT, *pCT;
			pKT = reinterpret_cast<__int64 *>(&KernelTime);
			pUT = reinterpret_cast<__int64 *>(&UserTime);
			pCT = reinterpret_cast<__int64 *>(&CreationTime);
			if(i == 0)
			{
				_tprintf( _T("cold start times:\nStartupTime %d ms"), result);
				_tprintf( _T(", PreCreationTime: %u ms"), ((*pCT)- wft)/ 10000); 
				_tprintf( _T(", KernelTime: %u ms"), (*pKT) / 10000);
				_tprintf( _T(", UserTime: %u ms\n"), (*pUT) / 10000); 
				_tprintf( _T("Waiting for statistics for %d warm samples"), COUNT); 
			}
			else
			{
				_tprintf( _T("."));
				kernelTimes[i-1] = (int)((*pKT) / 10000);
				preCreationTimes[i-1] = (int)((*pCT)- wft)/ 10000; 
				userTimes[i-1] = (int)((*pUT) / 10000);
				strtupTimes[i-1] = result;	
			}
		}
		else
		{
			printf( "GetProcessTimes failed for %p",  pi.hProcess ); 
		}
		// Close process and thread handles. 
		CloseHandle( pi.hProcess );
		CloseHandle( pi.hThread );
		if((int)result < 0)
		{
			_tprintf( _T("%s failed with code %d: %s\n"),cmd, result,GetErrorDescription(result) );
			return result;
		}
		::Sleep(1000); //1s delay

	}//for ends
	if(i <= COUNT )
	{
		result = GetLastError();
		_tprintf( _T("\nThere was an error while running '%s', last error code = %d: %s\n"),cmd,result,GetErrorDescription(result));
		return result;
	}

	double median, mean, stddev;
	if(CalculateStatistics(&strtupTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("\nStartupTime: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&preCreationTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("PreCreation: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&kernelTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("KernelTime : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&userTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("UserTime   : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}

	return GetLastError();
}

DWORD BenchMarkMemory( LPCTSTR szcProg)
{
	TCHAR cmd[100];
	STARTUPINFO si;
	PROCESS_INFORMATION pi;
	ZeroMemory( &si, sizeof(si) );
	si.cb = sizeof(si);
	ZeroMemory( &pi, sizeof(pi) );
	DWORD dwerr;
	if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("javaw -client -cp .\\.. %s"), szcProg);
	}
	else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("mono %s"), szcProg);
	}
	else
	{
			_stprintf_s(cmd,100,_T("%s"), szcProg);		 
	}
	::SetLastError(0);
	// Start the child process. 
	if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
	{
		dwerr = GetLastError();
		_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
		return dwerr;
	}

	//wait to show up
	::Sleep(3000);

    PROCESS_MEMORY_COUNTERS_EX pmc;
    if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
    {
		printf( "Normal size->PrivateUsage: %lu KB,",  pmc.PrivateUsage/1024 );
        printf( " Current WorkingSet: %lu KB\n", pmc.WorkingSetSize/1024 );
    }
	else
	{
		printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 

	}

	HWND hwnd = MinimizeUIApp(pi);
	//wait to minimize
	::Sleep(2000);

    if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
    {
		printf( "Minimized->  PrivateUsage: %lu KB,",  pmc.PrivateUsage/1024 );
        printf( " Current WorkingSet: %lu KB\n", pmc.WorkingSetSize/1024 );
    }
	else
		printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 
	

	if(!EmptyWorkingSet(pi.hProcess))
		printf( "EmptyWorkingSet failed for %x\n", pi.dwProcessId );
	else
	{
		ZeroMemory(&pmc, sizeof(pmc));
		if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
		{
		
			printf( "Minimum WorkingSet:        %lu KB,", pmc.WorkingSetSize/1024 );
			printf( " PeakWorkingSet:     %lu KB\n",  pmc.PeakWorkingSetSize/1024 );  
		}
		else
			printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 
	}

  //	if(!SetProcessWorkingSetSize(pi.hProcess, -1, -1))
		//printf( "SetProcessWorkingSetSize failed\n" );
	if(hwnd)
		::SendMessage(hwnd,WM_CLOSE,NULL,NULL);

	// Wait up to 20 seconds or until the child process exits.
	dwerr = WaitForSingleObject( pi.hProcess, 20000 );
	if(dwerr != WAIT_OBJECT_0)
	{
		dwerr = GetLastError();
		_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
	}
	// Close process and thread handles. 
	CloseHandle( pi.hProcess );
	CloseHandle( pi.hThread );
	return GetLastError();
}
void _tmain( int argc, TCHAR *argv[] )
{

    if( argc < 2 )
    {
        _tprintf( _T("Usage: %s [test1.exe test2.exe java_class....]\n"), argv[0]);
        return;
    }

	for(int i = 1; i < argc; i++)
	{
		_tprintf( _T("Benchmarking %s\n"), argv[i] );
		if(!BenchMarkTimes(argv[i]))
			BenchMarkMemory(argv[i]);
		_tprintf( _T("________________________________________________________________________________\n"));
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

dmihailescu
Software Developer (Senior)
United States United States
Decebal Mihailescu is a software engineer with interest in .Net, C# and C++.

| Advertise | Privacy | Terms of Use | Mobile
Web03 | 2.8.141223.1 | Last Updated 2 Sep 2010
Article Copyright 2010 by dmihailescu
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid