Click here to Skip to main content
Click here to Skip to main content
Add your own
alternative version

Benchmark start-up and system performance for .Net, Mono, Java, C++ and their respective UI

, 2 Sep 2010 CPOL
What is the start-up and system performance overhead for .Net, Mono, Java versus C++ and Forms, WPF, Swing versus MFC
BenchMarkStartup_src.zip
BenchMarkStartup
BenchMarkStartup
BenchMarkStartup.vcxproj.user
BenchMarkStartupUI
BenchMarkStartupUI.vcxproj.user
CPPMFCPerf
CPPMFCPerf.vcxproj.user
res
CPPMFCPerf.ico
Debug
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
DotNet11Perf
App.ico
DotNet11UIPerf
App.ico
DotNet20Perf
Properties
DotNet20UIPerf
Properties
Settings.settings
DotNet35Perf
Properties
DotNet35UIPerf
Properties
Settings.settings
DotNet35WPFPerf
Properties
Settings.settings
DotNet40Perf
Properties
DotNet40UIPerf
Properties
Settings.settings
DotNet40WPFPerf
Properties
Settings.settings
JavaPerf
StartupTest.class
SwingTest.class
MonoPerf
Mono26Perf.exe
Mono26UIPerf.exe
NativeCPPPerf
Release
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
NativeCPPPerf.exe
Release.zip
MonoPerf
Mono26Perf.exe
Mono26UIPerf.exe
Release
BenchMarkStartup.exe
BenchMarkStartupUI.exe
CPPMFCPerf.exe
DotNet11Perf.exe
DotNet11UIPerf.exe
DotNet20Perf.exe
DotNet20UIPerf.exe
DotNet35Perf.exe
DotNet35UIPerf.exe
DotNet35WPFPerf.exe
DotNet40Perf.exe
DotNet40UIPerf.exe
DotNet40WPFPerf.exe
mfc100u.dll
msvcm90.dll
msvcp100.dll
msvcp90.dll
msvcr100.dll
msvcr90.dll
NativeCPPPerf.exe
JavaPerf
StartupTest.class
SwingTest.class
// BenchMarkStartup.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include <windows.h>
#include <assert.h>
#include "statistics.h"

#include <psapi.h>
#pragma comment(lib, "Psapi.lib")
#include <atlstr.h>

BOOL PrintModules(const PROCESS_INFORMATION& pi )
{
    HMODULE hMods[1024];
    HANDLE hProcess;
    DWORD cbNeeded;
    unsigned int i;

    // Get a list of all the modules in this process.

    hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |
                            PROCESS_VM_READ,
                            FALSE, pi.dwProcessId );
    if (NULL == hProcess)
	{
		printf( "Failed to Open Process ID: 0x%08x\n", pi.dwProcessId);
        return FALSE;
	}

    if( EnumProcessModules(hProcess, hMods, sizeof(hMods), &cbNeeded))
    {
		// Print the process modules.
		UINT count  = cbNeeded / sizeof(HMODULE);
		printf( "Process ID: 0x%08x has %d modules:\n", pi.dwProcessId,count );
        for ( i = 0; i < count; i++ )
        {
            TCHAR szModName[MAX_PATH];

            // Get the full path to the module's file.

            if ( GetModuleFileNameEx( hProcess, hMods[i], szModName,
                                      sizeof(szModName) / sizeof(TCHAR)))
            {
                // Print the module name and handle value.

				CString strFile(szModName);
				int pathStart = strFile.GetLength() - strFile.ReverseFind('\\') -1 ;
				_tprintf( TEXT("\t%s (0x%08p)\n"), (pathStart > 0)?strFile.Right(pathStart):szModName, hMods[i] );
            }
			else
                _tprintf( TEXT("\t%s (0x%08p)\n"), TEXT("some protected module @"), hMods[i] );
        }
    }
	else
	{
		printf( "Failed to Enumerate modules for Process ID: 0x%08x\n", pi.dwProcessId);
	}

    CloseHandle( hProcess );
	return TRUE;
}

BOOL PrintMemoryInfo( const PROCESS_INFORMATION& pi)
{
  //wait 2 seconds while the process is sleeping for 5 seconds
  if(WAIT_TIMEOUT != WaitForSingleObject( pi.hProcess, 2000 ))
	  return FALSE;
  	if(!EmptyWorkingSet(pi.hProcess))
		printf( "EmptyWorkingSet failed for %x\n", pi.dwProcessId );
  //	if(!SetProcessWorkingSetSize(pi.hProcess, -1, -1))
		//printf( "SetProcessWorkingSetSize failed\n" );
	BOOL bres = TRUE;
    PROCESS_MEMORY_COUNTERS_EX pmc;
    if ( GetProcessMemoryInfo( pi.hProcess, (PROCESS_MEMORY_COUNTERS*)&pmc, sizeof(pmc)) )
    {
		printf( "PrivateUsage: %lu KB,",  pmc.PrivateUsage/1024 );
        printf( " Minimum WorkingSet: %lu KB,", pmc.WorkingSetSize/1024 );
        printf( " PeakWorkingSet: %lu KB\n",  pmc.PeakWorkingSetSize/1024 );  
        
#ifdef DEBUG
        printf( "PageFaultCount: %lu\n", pmc.PageFaultCount );
        printf( "QuotaPeakPagedPoolUsage: %lu\n",  pmc.QuotaPeakPagedPoolUsage );                 
        printf( "QuotaPagedPoolUsage: %lu\n",  pmc.QuotaPagedPoolUsage );
        printf( "QuotaPeakNonPagedPoolUsage: %lu\n",   pmc.QuotaPeakNonPagedPoolUsage );
        printf( "QuotaNonPagedPoolUsage: %lu\n",  pmc.QuotaNonPagedPoolUsage );
        printf( "PagefileUsage: %lu\n", pmc.PagefileUsage ); 
        printf( "PeakPagefileUsage: %lu\n",  pmc.PeakPagefileUsage ); 
#endif
    }
	else
	{
		printf( "GetProcessMemoryInfo failed for %p",  pi.hProcess ); 
		bres = FALSE;
	}

#ifdef DEBUG
	bres = PrintModules(pi) && bres;
#endif

	return bres;
}

BOOL PrepareColdStart()
{
    // Get the list of process identifiers.
	if(!EmptyWorkingSet(GetCurrentProcess()))
	{
		printf( "EmptyWorkingSet() failed for BenchMarkStartup.exe, PID = %x\n",::GetCurrentProcessId() );
		return FALSE;
	}

    DWORD aProcesses[1024], cbNeeded, cProcesses;
    unsigned int i;

    if ( !EnumProcesses( aProcesses, sizeof(aProcesses), &cbNeeded ) )
        return FALSE;

    // Calculate how many process identifiers were returned.

    cProcesses = cbNeeded / sizeof(DWORD);

    // Print the memory usage for each process

    for ( i = 0; i < cProcesses; i++ )
	{
		if(aProcesses[i] == 0)
			continue;
		HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION |PROCESS_SET_QUOTA,
								FALSE, aProcesses[i] );

		if (NULL == hProcess)
		{
			//printf( "Failed to Open Process ID: 0x%08x\n", aProcesses[i]);
			continue;
		}
		else
		{
			
			if(!EmptyWorkingSet(hProcess))
				printf( "EmptyWorkingSet failed for PID %x\n", aProcesses[i] );
			CloseHandle( hProcess );
		}

	}//for ends
	return TRUE;
}

__int64 currentJavaFileTime()
{
	static const __int64 startEpoch = 116444736000000000; // 1970/1/1 vs 1601/1/1
 //see comments below 

  FILETIME   ft;
  __int64 *pcrt;
#ifdef _DEBUG
 SYSTEMTIME st;
 st.wYear = 1970;
 st.wMonth = 1;
 st.wDay =1;
 st.wHour = 0;
 st.wMinute = 0;
 st.wSecond = 0;
 st.wMilliseconds = 0;
 BOOL bres = ::SystemTimeToFileTime(&st,&ft); 
  pcrt = reinterpret_cast<__int64 *>(&ft);
  assert(*pcrt == startEpoch);
#endif
  //Retrieves the current system date and time. The information is in Coordinated Universal Time (UTC) format.
  //resolutin 100ns
  GetSystemTimeAsFileTime(&ft);

  pcrt = reinterpret_cast<__int64 *>(&ft);
  return (*pcrt - startEpoch); 
}

__int64 currentWindowsFileTime()
{
	static const __int64 startEpoch2 = 0; // 1601/1/1
 //see comments below 
  FILETIME   ft;
  __int64 *pcrt;
#ifdef _DEBUG
 SYSTEMTIME st;
 st.wYear = 1601;
 st.wMonth = 1;
 st.wDay =1;
 st.wHour = 0;
 st.wMinute = 0;
 st.wSecond = 0;
 st.wMilliseconds = 0;
 BOOL bres = ::SystemTimeToFileTime(&st,&ft); 
  pcrt = reinterpret_cast<__int64 *>(&ft);
  assert(*pcrt == startEpoch2);
#endif
/*1 January 1601 to AD 30,828
*/
  GetSystemTimeAsFileTime(&ft);
  pcrt = reinterpret_cast<__int64 *>(&ft);
  return (*pcrt - startEpoch2); 
}

TCHAR strErr[200];
LPCTSTR GetErrorDescription(INT code)
{
	if(!code)
		return _T("Success");
	TCHAR* lpMsgBuf = NULL;
	FormatMessage( 
		FORMAT_MESSAGE_ALLOCATE_BUFFER | 
		FORMAT_MESSAGE_FROM_SYSTEM | 
		FORMAT_MESSAGE_IGNORE_INSERTS,
		NULL,
		code,
		0, // Default language
		(LPTSTR) &lpMsgBuf,
		0,
		NULL);

	if(lpMsgBuf)
	{
		//g_strErr = lpMsgBuf;
		_tcscpy_s(strErr,200, lpMsgBuf);
		LocalFree(lpMsgBuf);
	}
	else
		return _T("Unknown Error");
	return strErr;
}

#ifdef _DEBUG
const int COUNT = 3;
#else
const int COUNT = 9;
#endif

int strtupTimes[COUNT];
int kernelTimes[COUNT];
int preCreationTimes[COUNT];
int userTimes[COUNT];

DWORD BenchMarkTimes( LPCTSTR szcProg)
{
	ZeroMemory( strtupTimes, sizeof(strtupTimes) );
	ZeroMemory( kernelTimes, sizeof(kernelTimes) );
	ZeroMemory( preCreationTimes, sizeof(preCreationTimes) );
	ZeroMemory( userTimes, sizeof(userTimes) );
	BOOL res = TRUE;
	TCHAR cmd[100];
	int i,result = 0;
	DWORD dwerr = 0;
	PrepareColdStart();
	::Sleep(3000);//3 seconds delay

	for(i = 0; i <= COUNT && res; i++)
	{
		STARTUPINFO si;
		PROCESS_INFORMATION pi;
		ZeroMemory( &si, sizeof(si) );
		si.cb = sizeof(si);
		ZeroMemory( &pi, sizeof(pi) );
		::SetLastError(0);
		__int64 wft = 0;
		if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
		{ //java -client -cp .\.. JavaPerf.StartupTest
			wft = currentWindowsFileTime();
			_stprintf_s(cmd,100,_T("java -client -cp .\\.. %s \"%I64d\""), szcProg,wft);
		}
		else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
		{ //mono ..\monoperf\mono26perf.exe
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("mono %s \"%I64d\""), szcProg,wft);
		}
		else
		{
				wft = currentWindowsFileTime();
				_stprintf_s(cmd,100,_T("%s \"%I64d\""), szcProg,wft);		 
		}

		// Start the child process. 
		if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
		{
			dwerr = GetLastError();
			_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
			return dwerr;
		}

		// Wait up to 20 seonds or until the child process exits.
		dwerr = WaitForSingleObject( pi.hProcess, 20000 );
		if(dwerr != WAIT_OBJECT_0)
		{
			dwerr = GetLastError();
			_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
			// Close process and thread handles. 
			CloseHandle( pi.hProcess );
			CloseHandle( pi.hThread );
			break;
		}
		res = GetExitCodeProcess(pi.hProcess,(LPDWORD)&result);
		FILETIME CreationTime,ExitTime,KernelTime,UserTime;
		if(GetProcessTimes(pi.hProcess,&CreationTime,&ExitTime,&KernelTime,&UserTime))
		{
			__int64 *pKT,*pUT, *pCT;
			pKT = reinterpret_cast<__int64 *>(&KernelTime);
			pUT = reinterpret_cast<__int64 *>(&UserTime);
			pCT = reinterpret_cast<__int64 *>(&CreationTime);
			if(i == 0)
			{
				_tprintf( _T("cold start times:\nStartupTime %d ms"), result);
				_tprintf( _T(", PreCreationTime: %u ms"), ((*pCT)- wft)/ 10000); 
				_tprintf( _T(", KernelTime: %u ms"), (*pKT) / 10000);
				_tprintf( _T(", UserTime: %u ms\n"), (*pUT) / 10000); 
				_tprintf( _T("Waiting for statistics for %d warm samples"), COUNT); 
			}
			else
			{
				_tprintf( _T("."));
				kernelTimes[i-1] = (int)((*pKT) / 10000);
				preCreationTimes[i-1] = (int)((*pCT)- wft)/ 10000; 
				userTimes[i-1] = (int)((*pUT) / 10000);
				strtupTimes[i-1] = result;	
			}
		}
		else
		{
			printf( "GetProcessTimes failed for %p",  pi.hProcess ); 
		}
		// Close process and thread handles. 
		CloseHandle( pi.hProcess );
		CloseHandle( pi.hThread );
		if((int)result < 0)
		{
			_tprintf( _T("%s failed with code %d: %s\n"),cmd, result,GetErrorDescription(result) );
			return result;
		}
		::Sleep(1000); //1s delay

	}//for ends
	if(i <= COUNT )
	{
		_tprintf( _T("\nThere was an error while running '%s', last error code = %d\n"),cmd,GetLastError());
		return result;
	}

	double median, mean, stddev;
	if(CalculateStatistics(&strtupTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("\nStartupTime: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&preCreationTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("PreCreation: mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&kernelTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("KernelTime : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}
	if(CalculateStatistics(&userTimes[0], COUNT, median,  mean,  stddev))
	{
		_tprintf( _T("UserTime   : mean = %6.2f ms, median = %3.0f ms, standard deviation = %6.2f ms\n"), 
			mean,median,stddev);
	}

	return GetLastError();
}

DWORD BenchMarkMemory( LPCTSTR szcProg)
{

	TCHAR cmd[100];
	STARTUPINFO si;
	PROCESS_INFORMATION pi;
	ZeroMemory( &si, sizeof(si) );
	si.cb = sizeof(si);
	ZeroMemory( &pi, sizeof(pi) );
	DWORD dwerr;
	if(StrStrI(szcProg, _T("java")) && !StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("java -client -cp .\\.. %s"), szcProg);
	}
	else if(StrStrI(szcProg, _T("mono")) && StrStrI(szcProg, _T(".exe")))
	{
		_stprintf_s(cmd,100,_T("mono %s"), szcProg);
	}
	else
	{
			_stprintf_s(cmd,100,_T("%s"), szcProg);		 
	}
	::SetLastError(0);
	// Start the child process. 
	if( !CreateProcess( NULL,cmd,NULL,NULL,FALSE,0,NULL,NULL,&si,&pi )) 
	{
		dwerr = GetLastError();
		_tprintf( _T("CreateProcess failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
		return dwerr;
	}

	if(!PrintMemoryInfo(pi))
	{
		dwerr = GetLastError();
		_tprintf( _T("PrintMemoryInfo failed for '%s' with error code %d:%s.\n"),szcProg, dwerr,GetErrorDescription(dwerr) );
	}
	// Wait up to 20 seonds or until the child process exits.
	dwerr = WaitForSingleObject( pi.hProcess, 20000 );
	if(dwerr != WAIT_OBJECT_0)
	{
		dwerr = GetLastError();
		_tprintf( _T("WaitForSingleObject failed for '%s' with error code %d\n"),szcProg, dwerr );
	}
	// Close process and thread handles. 
	CloseHandle( pi.hProcess );
	CloseHandle( pi.hThread );
	return GetLastError();
}
void _tmain( int argc, TCHAR *argv[] )
{

    if( argc < 2 )
    {
        _tprintf( _T("Usage: %s [test1.exe test2.exe java_class....]\n"), argv[0]);
        return;
    }

	for(int i = 1; i < argc; i++)
	{
		_tprintf( _T("Benchmarking %s\n"), argv[i] );
		BenchMarkTimes(argv[i]);
		BenchMarkMemory(argv[i]);
		_tprintf( _T("________________________________________________________________________________\n"));
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Share

About the Author

dmihailescu
Software Developer (Senior)
United States United States
Decebal Mihailescu is a software engineer with interest in .Net, C# and C++.

| Advertise | Privacy | Terms of Use | Mobile
Web02 | 2.8.141220.1 | Last Updated 2 Sep 2010
Article Copyright 2010 by dmihailescu
Everything else Copyright © CodeProject, 1999-2014
Layout: fixed | fluid