Tweaking WCF to build highly scalable async REST API

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net;
using System.Web;
using System.IO;
using System.Configuration;

namespace LoadTestProxy
{
    class Program
    {
        readonly static int MAX_THREAD_COUNT = int.Parse(ConfigurationManager.AppSettings["MAX_THREAD_COUNT"]);
        readonly static int ATTEMPTS = int.Parse(ConfigurationManager.AppSettings["ATTEMPTS"]);
        readonly static string WcfServiceHost = ConfigurationManager.AppSettings["WcfServiceHost"]; // "132.146.124.132/WcfAsyncRestApi"; // "localhost:8080";  // 
        readonly static string SomeServiceHost = ConfigurationManager.AppSettings["SomeServiceHost"]; // "localhost:8000";//  

        static readonly string EchoUrl = HttpUtility.UrlEncode(string.Format(ConfigurationManager.AppSettings["EchoUrl"], SomeServiceHost));
        static readonly string AsyncServiceUrl = string.Format(
            ConfigurationManager.AppSettings["AsyncServiceUrl"],
            EchoUrl, 0, WcfServiceHost);
        static readonly string RegularServiceUrl = string.Format(
            ConfigurationManager.AppSettings["SyncServiceUrl"],
            EchoUrl, 0, WcfServiceHost);
        static readonly string ResponsivenessTestUrl = string.Format(ConfigurationManager.AppSettings["ResponsivenessUrl"], WcfServiceHost);

        static void Main(string[] args)
        {
            var serviceResponseTimesForAsync = new List<TimeSpan>();
            var aspnetResponseTimesForAsync = new List<TimeSpan>();

            var serviceResponseTimesForRegular = new List<TimeSpan>();
            var aspnetResponseTimesForRegular = new List<TimeSpan>();

            int asyncSlowASPNETResponseCount = 0;
            int regularSlowASPNETResponseCount = 0;

            Console.WriteLine("Warming up Services");
            Console.WriteLine("================================");
            HitService(AsyncServiceUrl, ResponsivenessTestUrl, 2, new TimeSpan[MAX_THREAD_COUNT/2], new TimeSpan[MAX_THREAD_COUNT/2], out asyncSlowASPNETResponseCount, "[ASYNC]");
            HitService(RegularServiceUrl, ResponsivenessTestUrl, 2, new TimeSpan[MAX_THREAD_COUNT / 2], new TimeSpan[MAX_THREAD_COUNT / 2], out regularSlowASPNETResponseCount, "[SYNC]");
            Thread.Sleep(10000);

            asyncSlowASPNETResponseCount = 0;
            asyncSlowASPNETResponseCount = 0;

            TimeSpan[] asyncServiceAvgResponseTimes = new TimeSpan[ATTEMPTS];
            TimeSpan[] regularServiceAvgResponseTimes = new TimeSpan[ATTEMPTS];            
            for (int i = 0; i < ATTEMPTS; i++)
            {
                HitAsyncService(serviceResponseTimesForAsync, aspnetResponseTimesForAsync, asyncServiceAvgResponseTimes, i, out asyncSlowASPNETResponseCount);
                Thread.Sleep(10000);
                HitRegularService(serviceResponseTimesForRegular, aspnetResponseTimesForRegular, regularServiceAvgResponseTimes, i, out regularSlowASPNETResponseCount);
                Thread.Sleep(10000);
            }

            // Compare slow responses
            Console.WriteLine("Regular service slow responses: {0}", regularSlowASPNETResponseCount);
            Console.WriteLine("Async service slow responses: {0}", asyncSlowASPNETResponseCount);
            
            // Calculate average service response time
            CalculateAndCompareServiceResponseTimes(asyncServiceAvgResponseTimes, regularServiceAvgResponseTimes);

            // Calculate average ASP.NET response time
            CalculateAndCompareASPNETResponseTimes(aspnetResponseTimesForAsync, aspnetResponseTimesForRegular);

            ComparePercentilePerformance(95, serviceResponseTimesForAsync, aspnetResponseTimesForAsync, serviceResponseTimesForRegular, aspnetResponseTimesForRegular);

            File.WriteAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AsyncServiceResponseTimes.txt"),
                serviceResponseTimesForAsync.ConvertAll<string>(t => t.TotalSeconds.ToString()).ToArray());
            File.WriteAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "RegularServiceResponseTimes.txt"),
                serviceResponseTimesForRegular.ConvertAll<string>(t => t.TotalSeconds.ToString()).ToArray());
            File.WriteAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "ASPNETAsyncResponseTimes.txt"),
                aspnetResponseTimesForAsync.ConvertAll<string>(t => t.TotalSeconds.ToString()).ToArray());
            File.WriteAllLines(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "ASPNETRegularResponseTimes.txt"),
                aspnetResponseTimesForRegular.ConvertAll<string>(t => t.TotalSeconds.ToString()).ToArray());

            Console.ReadLine();
        }

        private static void ComparePercentilePerformance(double percentile, List<TimeSpan> serviceResponseTimesForAsync, List<TimeSpan> aspnetResponseTimesForAsync, List<TimeSpan> serviceResponseTimesForRegular, List<TimeSpan> aspnetResponseTimesForRegular)
        {
            // Calculate 90%ile for async service and ASP.NET response during async service calls
            var percentileServiceResponseTimeForAsync = CalculatePercentile(serviceResponseTimesForAsync.ConvertAll<double>(t => t.TotalSeconds), percentile);
            var percentileAspNetResponseTimeForAsync = CalculatePercentile(aspnetResponseTimesForAsync.ConvertAll<double>(t => t.TotalSeconds), percentile);
            Console.WriteLine("Async {1}%ile Service Response Time: {0}", percentileServiceResponseTimeForAsync, percentile);
            Console.WriteLine("Async {1}%ile ASP.NET Response Time: {0}", percentileAspNetResponseTimeForAsync, percentile);

            // Calculate 95%ile for regular service and ASP.NET response during regular service calls
            var percentileServiceResponseTimeForRegular = CalculatePercentile(serviceResponseTimesForRegular.ConvertAll<double>(t => t.TotalSeconds), percentile);
            var percentileAspNetResponseTimeForRegular = CalculatePercentile(aspnetResponseTimesForRegular.ConvertAll<double>(t => t.TotalSeconds), percentile);
            Console.WriteLine("Regular {1}%ile Service Response Time: {0}", percentileServiceResponseTimeForRegular, percentile);
            Console.WriteLine("Regular {1}%ile ASP.NET Response Time: {0}", percentileAspNetResponseTimeForRegular, percentile);

            // Compare ASP.NET response time during async vs regular service 
            if (percentileAspNetResponseTimeForAsync < percentileAspNetResponseTimeForRegular)
                Console.WriteLine("{1}%ile ASP.NET Response time is better for Async by {0}%",
                    100 - (percentileAspNetResponseTimeForAsync / percentileAspNetResponseTimeForRegular * 100), percentile);
            else
                Console.WriteLine("{1}%ile ASP.NET Response time is worse for Async by {0}%",
                    100 - (percentileAspNetResponseTimeForRegular / percentileAspNetResponseTimeForAsync * 100), percentile);

            // Compare service response time during async vs regular service call
            if (percentileServiceResponseTimeForAsync < percentileServiceResponseTimeForRegular)
                Console.WriteLine("{1}%ile Service Response time is better for Async by {0}%",
                    100 - (percentileServiceResponseTimeForAsync / percentileServiceResponseTimeForRegular * 100), percentile);
            else
                Console.WriteLine("{1}%ile Service Response time is worse for Async by {0}%",
                    100 - (percentileServiceResponseTimeForRegular / percentileServiceResponseTimeForAsync * 100), percentile);
        }

        private static void CalculateAndCompareASPNETResponseTimes(List<TimeSpan> aspnetResponseTimesForAsync, List<TimeSpan> aspnetResponseTimesForRegular)
        {
            var averageAspnetResponseTimesForRegular = aspnetResponseTimesForRegular.Average(t => t.TotalSeconds);
            var averageAspnetResponseTimeForAsync = aspnetResponseTimesForAsync.Average(t => t.TotalSeconds);

            Console.WriteLine("Regular ASP.NET average response time: {0}", averageAspnetResponseTimesForRegular);
            Console.WriteLine("Async ASP.NET average response time: {0}", averageAspnetResponseTimeForAsync);

            if (averageAspnetResponseTimesForRegular < averageAspnetResponseTimeForAsync)
                Console.WriteLine("Async ASP.NET is {0}% slower.",
                    100 - (averageAspnetResponseTimesForRegular / averageAspnetResponseTimeForAsync * 100));
            else
                Console.WriteLine("Async ASP.NET is {0}% faster.",
                    100 - (averageAspnetResponseTimeForAsync / averageAspnetResponseTimesForRegular * 100));
        }

        private static void CalculateAndCompareServiceResponseTimes(TimeSpan[] asyncServiceAvgResponseTimes, TimeSpan[] regularServiceAvgResponseTimes)
        {
            var regularServiceAvgResponseTime = regularServiceAvgResponseTimes.Average(t => t.TotalSeconds);
            var asyncServiceAvgResponseTime = asyncServiceAvgResponseTimes.Average(t => t.TotalSeconds);

            Console.WriteLine("Regular service average response time: {0}", regularServiceAvgResponseTime);
            Console.WriteLine("Async service average response time: {0}", asyncServiceAvgResponseTime);

            if (regularServiceAvgResponseTime < asyncServiceAvgResponseTime)
                Console.WriteLine("Async service is {0}% slower.",
                    100 - (regularServiceAvgResponseTime / asyncServiceAvgResponseTime * 100));
            else
                Console.WriteLine("Async service is {0}% faster.",
                    100 - (asyncServiceAvgResponseTime / regularServiceAvgResponseTime * 100));
        }

        private static void HitRegularService(List<TimeSpan> serviceResponseTimesForRegular, List<TimeSpan> aspnetResponseTimesForRegular, TimeSpan[] regularServiceAvgResponseTimes, int i, out int regularSlowASPNETResponseCount)
        {
            Console.WriteLine("Testing Regular Service ");
            Console.WriteLine("================================");
            var serviceResponseTimesForRegularAttempt = new TimeSpan[MAX_THREAD_COUNT / 2];
            var aspnetResponseTimesForRegularAttmpt = new TimeSpan[MAX_THREAD_COUNT / 2];
            regularServiceAvgResponseTimes[i] = HitService(RegularServiceUrl, ResponsivenessTestUrl, MAX_THREAD_COUNT, serviceResponseTimesForRegularAttempt, aspnetResponseTimesForRegularAttmpt, out regularSlowASPNETResponseCount, "[SYNC]");
            serviceResponseTimesForRegular.AddRange(serviceResponseTimesForRegularAttempt);
            aspnetResponseTimesForRegular.AddRange(aspnetResponseTimesForRegularAttmpt);
        }

        private static void HitAsyncService(List<TimeSpan> serviceResponseTimesForAsync, List<TimeSpan> aspnetResponseTimesForAsync, TimeSpan[] asyncServiceAvgResponseTimes, int i, out int asyncSlowASPNETResponseCount)
        {
            Console.WriteLine("Testing Async Service");
            Console.WriteLine("================================");
            var serviceResponseTimesForAsyncAttemp = new TimeSpan[MAX_THREAD_COUNT / 2];
            var aspnetResponseTimesForAsyncAttemp = new TimeSpan[MAX_THREAD_COUNT / 2];
            asyncServiceAvgResponseTimes[i] = HitService(AsyncServiceUrl, ResponsivenessTestUrl, MAX_THREAD_COUNT, serviceResponseTimesForAsyncAttemp, aspnetResponseTimesForAsyncAttemp, out asyncSlowASPNETResponseCount, "[ASYNC]");
            serviceResponseTimesForAsync.AddRange(serviceResponseTimesForAsyncAttemp);
            aspnetResponseTimesForAsync.AddRange(aspnetResponseTimesForAsyncAttemp);
        }

        //static double CalculateAverage(string url, string responsivenessUrl, int attempts, List<TimeSpan> serviceResponseTimes, List<TimeSpan> aspnetResponseTimes)
        //{
        //    TimeSpan[] durations = new TimeSpan[attempts];
        //    for (int i = 0; i < durations.Length; i++)
        //    {
        //        Console.WriteLine("Test {0}", i);
        //        Console.WriteLine("--------------");
        //        durations[i] = HitService(url, responsivenessUrl, serviceResponseTimes, aspnetResponseTimes);
        //    }

        //    Console.Write("Average duration: ");
        //    var average = durations.Sum(t => t.TotalSeconds) / durations.Length;
        //    Console.WriteLine(average);
        //    return average;
        //}
        static TimeSpan HitService(string url, 
            string responsivenessUrl, 
            int threadCount, 
            TimeSpan[] serviceResponseTimes, 
            TimeSpan[] aspnetResponseTimes, 
            out int slowASPNETResponseCount,
            string logPrefix)
        {            
            Thread[] threads = new Thread[threadCount];
            var serviceResponseTimesCount = 0;
            var aspnetResponseTimesCount = 0;
            var slowCount = 0;
            var startTime = DateTime.Now;
            var serviceThreadOrderNo = 0;
            var aspnetThreadOrderNo = 0;
            for (int i = 0; i < threadCount/2; i++)
            {
                Thread aThread = new Thread(new ThreadStart(() =>
                    {
                        using (WebClient client = new WebClient())
                        {
                            try
                            {
                                var start = DateTime.Now;
                                Console.WriteLine("{0}\t{1}\t{2} Service call Start", logPrefix, serviceThreadOrderNo,
                                    Thread.CurrentThread.ManagedThreadId);
                                var content = client.DownloadString(url);
                                var duration = DateTime.Now - start;
                                lock (serviceResponseTimes)
                                {
                                    serviceResponseTimes[serviceResponseTimesCount++] = duration;
                                    Console.WriteLine("{0}\t{1}\t{2} End Service call. Duration: {2}", 
                                        logPrefix, serviceThreadOrderNo,
                                        Thread.CurrentThread.ManagedThreadId, duration.TotalSeconds);

                                    serviceThreadOrderNo++;
                                }
                                    
                            }
                            catch (Exception x)
                            {
                                Console.WriteLine(x);
                            }
                        }
                    }));
                aThread.Start();
                threads[i] = aThread;
            }

            // Give chance to start the calls
            Thread.Sleep(500);

            for (int i = threadCount / 2; i < threadCount; i ++)
            {
                Thread aThread = new Thread(new ThreadStart(() =>
                    {
                        using (WebClient client = new WebClient())
                        {

                            try
                            {
                                var start = DateTime.Now;
                                Console.WriteLine("{0}\t{1}\t{2} ASP.NET Page Start", logPrefix, aspnetThreadOrderNo,
                                    Thread.CurrentThread.ManagedThreadId);
                                var content = client.DownloadString(responsivenessUrl);
                                var duration = DateTime.Now - start;

                                lock (aspnetResponseTimes)
                                {
                                    aspnetResponseTimes[aspnetResponseTimesCount++] = duration;
                                    Console.WriteLine("{0}\t{1}\t{2} End of ASP.NET Call. Duration: {3}", 
                                        logPrefix, aspnetThreadOrderNo,
                                        Thread.CurrentThread.ManagedThreadId, duration.TotalSeconds);
                                    aspnetThreadOrderNo++;
                                }
                                if (serviceResponseTimesCount > 0)
                                {
                                    Console.WriteLine("{0} WARNING! ASP.NET requests running slower than service.", logPrefix);
                                    slowCount++;
                                }
                            }
                            catch (Exception x)
                            {
                                Console.WriteLine(x);
                            }
                        }
                    }));
                aThread.Start();
                threads[i] = aThread;
            }
            
            // wait for all threads to finish execution
            foreach (Thread thread in threads)
                thread.Join();
            
            var endTime = DateTime.Now;
            var totalDuration = endTime - startTime;
            Console.WriteLine(totalDuration.TotalSeconds);
            slowASPNETResponseCount = slowCount;
            return totalDuration;
        }

        /// <summary>
        /// Calculate percentile of a sorted data set
        /// </summary>
        /// <param name="sortedData">array of double values</param>
        /// <param name="p">percentile, value 0-100</param>
        /// <returns></returns>
        internal static double CalculatePercentile(List<double> list, double p)
        {
            double[] sortedData = list.ToArray();
            Array.Sort(sortedData);

            // algo derived from Aczel pg 15 bottom
            if (p >= 100.0d) return sortedData[sortedData.Length - 1];

            double position = (double)(sortedData.Length + 1) * p / 100.0;
            double leftNumber = 0.0d, rightNumber = 0.0d;

            double n = p / 100.0d * (sortedData.Length - 1) + 1.0d;

            if (position >= 1)
            {
                leftNumber = sortedData[(int)System.Math.Floor(n) - 1];
                rightNumber = sortedData[(int)System.Math.Floor(n)];
            }
            else
            {
                leftNumber = sortedData[0]; // first data
                rightNumber = sortedData[1]; // first data
            }

            if (leftNumber == rightNumber)
                return leftNumber;
            else
            {
                double part = n - System.Math.Floor(n);
                return leftNumber + part * (rightNumber - leftNumber);
            }
        } // end of internal function percentile
    }
}