#include "stdafx.h"
#include <ctype.h>
#include <limits>
#include <algorithm>
#include "Misc.h"
#include "Benchmarker.h"
using namespace std;
////////////////////////////////////////////////////////////////////////////////
BenchmarkStatistic::BenchmarkStatistic()
{
Clear();
}
void BenchmarkStatistic::Clear()
{
Statistic::Clear();
First = Last = Errors = 0;
UserData.clear();
}
void BenchmarkStatistic::Add(double nextValue, const string& userDatum)
{
Last = nextValue;
if (Count == 0)
First = nextValue;
if (!userDatum.empty())
UserData.insert(userDatum);
if (!_finite(nextValue))
Errors++;
else
Statistic::Add(nextValue);
}
////////////////////////////////////////////////////////////////////////////////
const std::string Benchmarker::DiscardResult("(discard result)");
Benchmarker::Benchmarker()
{
DefaultNumTrials = 5;
UserDataColumnName = "Comment";
}
void Benchmarker::MeasureAndRecord(string name, FastDelegate0<string> code)
{
string oldActive = _activeBenchmark;
if (!_activeBenchmark.empty())
_activeBenchmark += ": " + name;
else
_activeBenchmark = name;
try {
string userData;
double time = Measure(code, OUT userData);
if (userData != DiscardResult)
Tally(_activeBenchmark, time, userData);
}
catch(exception& e)
{
++_errors.GetOrAdd(string(e.what()), 0);
TallyError(_activeBenchmark, typeid(e).name());
}
_activeBenchmark = oldActive;
}
double Benchmarker::Measure(FastDelegate0<string> code, OUT string& message)
{
SimpleTimer timer;
message = code();
double time = timer.Millisec() / 1000.0;
if (message.size() > 2 && message[0] == 'o' && message[1] == ':')
{
int overhead = atoi(&message[2]);
message.clear();
return time - overhead / 1000.0;
}
return time;
}
void Benchmarker::TallyError(const string& name, const string& excType)
{
Tally(name, numeric_limits<double>::quiet_NaN(), excType);
}
void Benchmarker::Tally(const string& name, double seconds, const string& userData)
{
// For once, this is easier in C++ than C#
_results[name].Add(seconds, userData);
}
void Benchmarker::RunAllBenchmarks(const vector< pair< string,FastDelegate0<string> > > &methods, bool randomOrder, FastDelegate0<> postprocess)
{
// Prepare the coallated order to do them in
vector< pair< string,FastDelegate0<string> > > order;
bool done = false;
for (int j = 0; !done; j++) {
done = true;
for (int i = 0; i < (int)methods.size(); i++)
{
// custom number of trials per method is not supported in C++
int trials = DefaultNumTrials;
if (j < trials) {
order.push_back(methods[i]);
done = false;
}
}
}
// Randomize if requested
if (randomOrder) {
for (int i = 0; i < (int)order.size(); i++)
{
int j = rand() % order.size();
std::swap(order[i], order[j]);
}
}
Clear();
// Finally, do the benchmarks
for (int i = 0; i < (int)order.size(); i++)
{
MeasureAndRecord(order[i].first, order[i].second);
postprocess();
}
}
void Benchmarker::RunAllBenchmarksInConsole(const vector< pair< string,FastDelegate0<string> > > &methods, bool randomOrder)
{
_rabicEnd = GetCursorTop();
FastDelegate0<> helper(this, &Benchmarker::RabicHelper);
RunAllBenchmarks(methods, randomOrder, helper);
SetCursorPosition(0, _rabicEnd);
}
void Benchmarker::RabicHelper()
{
int start = GetCursorTop();
PrintResults(stdout);
_rabicEnd = GetCursorTop();
SetCursorPosition(0, start);
}
void Benchmarker::Clear()
{
_errors.clear();
_results.clear();
}
void Benchmarker::PrintResults(FILE* writer, const string& separator, bool addPadding)
{
_printSeparator = separator;
FastDelegate1<vector<string>&, string> joiner(this, &Benchmarker::JoinStrings);
PrintResults(writer, separator, addPadding, joiner);
}
string Benchmarker::JoinStrings(vector<string>& list)
{
string out;
for (int i = 0; i < (int)list.size(); i++) {
if (i > 0)
out += _printSeparator;
out += list[i];
}
return out;
}
void Benchmarker::PrintResults(FILE* writer, const string& separator, bool addPadding, FastDelegate1<vector<string>&, string> userDataFormatter)
{
PrintResults(writer, Results(), separator, addPadding, userDataFormatter, UserDataColumnName);
// Print errors
if (_errors.size() > 0) {
printf("Errors occurred:");
map<string,int>::iterator it = _errors.begin();
for (; it != _errors.end(); ++it)
printf("%s x%d", it->first, it->second);
}
}
void Benchmarker::PrintResults(FILE* writer, const EasyMap<string, BenchmarkStatistic>& results, const string& separator,
bool addPadding, FastDelegate1<vector<string>&, string> userDataFormatter, const string& userDataColumnName)
{
map<string, BenchmarkStatistic>::const_iterator it;
int maxCount = 0;
bool haveUserData = false;
for (it = results.begin(); it != results.end(); ++it) {
maxCount = max(maxCount, it->second.Count);
haveUserData |= it->second.UserData.size() != 0;
}
// Prepare a list of columns
vector<ColInfo> columns;
columns.push_back(ColInfo("Test name", GetColumn(&PrTestName)));
columns.push_back(ColInfo("#", GetColumn(&PrCount)));
if (maxCount >= 2)
columns.push_back(ColInfo("Average", GetColumn(&PrAverage)));
columns.push_back(ColInfo("First", GetColumn(&PrFirst)));
if (maxCount >= 3) {
columns.push_back(ColInfo("Max", GetColumn(&PrMax)));
columns.push_back(ColInfo("Min", GetColumn(&PrMin)));
columns.push_back(ColInfo("Std.Dev", GetColumn(&PrStdDev)));
}
if (haveUserData) {
GetColumn gc(this, &Benchmarker::PrUserData);
columns.push_back(ColInfo(userDataColumnName, gc));
}
_userDataFormatter = userDataFormatter;
if (addPadding)
for (int i = 0; i < (int)columns.size(); i++) {
int max = (int)columns[i].ColName.length();
for (it = results.begin(); it != results.end(); ++it) {
string text = columns[i].Getter(it->first, it->second);
if (max < (int)text.length())
max = (int)text.length();
}
columns[i].Width = max;
}
// Now sort (case insensitively) and print the results
PrintRow(writer, columns, separator, NULL, NULL, GetColumn2(&PrColName));
vector< pair<string, BenchmarkStatistic> > results2(results.begin(), results.end());
sort(results2.begin(), results2.end(), &IsKeyLessCaseInsensitive<BenchmarkStatistic>);
for (size_t i = 0; i < results2.size(); i++)
PrintRow(writer, columns, separator, &results2[i].first, &results2[i].second, GetColumn2(&PrColGetter));
}
string Benchmarker::PrUserData(const string&, const BenchmarkStatistic& s)
{
vector<string> data;
for (set<string>::const_iterator it = s.UserData.begin(); it != s.UserData.end(); ++it)
data.push_back(*it);
return _userDataFormatter(data);
}
void Benchmarker::PrintRow(FILE* writer, vector<ColInfo>& columns, const string& separator, const string* benchmarkName, const BenchmarkStatistic* stats, GetColumn2 getter)
{
for (int i = 0; i < (int)columns.size(); i++)
{
if (i > 0)
fprintf(writer, "%s", separator.c_str());
string text = getter(benchmarkName, stats, columns[i]);
bool hasLetters = false;
for (int j = 0; j < (int)text.length(); j++)
if (isalpha(text[j]))
hasLetters = true;
int w = columns[i].Width;
fprintf(writer, "%*s", hasLetters ? -w : w, text.c_str());
}
fprintf(writer, "\n");
}