Measuring and Monitoring WCF Web Service Performance

using System;
using System.Collections.Generic;
using System.Net;
using System.Linq;
using System.Web;
using Kaleida.ServiceMonitor.Model.Email;
using Kaleida.ServiceMonitor.Model.Runtime;

namespace Kaleida.ServiceMonitor.Model
{
    internal class NotificationMessageBuilder
    {
        private const string HtmlColorSuccess = "#0A0";
        private const string HtmlColourFailure = "#A00";
        private const string HtmlColourFailureFresh = "#D00";

        public NotificationMessage BuildSummary(Monitor monitor, OperationStatisticsMap statsSinceLastSummary)
        {
            var html = new HtmlMessageBuilder();

            AppendInstanceInformation(html);
            AppendMonitoringPeriod(html, monitor);

            html.AddRawFormat("<p>The severity level is currently {0}</p>", monitor.State.ToSeverityLevel().ToSummaryHtml());

            AppendErrorSummary(html, monitor.State);

            AppendCombinedAvailabilitySummary(html, statsSinceLastSummary, "Summary <span style='font-size:smaller'>(since last report)</span>".AsRawHtml());
            AppendAvailabilityTable(html, statsSinceLastSummary, "Availability <span style='font-size:smaller'>(since last report)</span>".AsRawHtml());
            AppendPerformanceTable(html, statsSinceLastSummary, "Performance <span style='font-size:smaller'>(since last report)</span>".AsRawHtml());

            AppendCombinedAvailabilitySummary(html, monitor.Statistics, "Summary <span style='font-size:smaller'>(since monitoring began)</span>".AsRawHtml());
            AppendAvailabilityTable(html, monitor.Statistics, "Availability <span style='font-size:smaller'>(since monitoring began)</span>".AsRawHtml());
            AppendPerformanceTable(html, monitor.Statistics, "Performance <span style='font-size:smaller'>(since monitoring began)</span>".AsRawHtml());

            AppendRecentResponses(html, monitor);

            var subject = string.Format("Scheduled Summary for Script '{0}'", monitor.Workspace.CurrentScript.Name);
            return new NotificationMessage(subject, html.ToString());
        }

		public NotificationMessage BuildThresholdChangeSummary(Monitor monitor, SeverityLevel mostRecentSeverityLevel)
        {
            var html = new HtmlMessageBuilder();

            AppendInstanceInformation(html);
            AppendMonitoringPeriod(html, monitor);

            html.AddRawFormat("<p>The severity threshold has changed from {0} to {1}</p>", mostRecentSeverityLevel.ToSummaryHtml(), monitor.State.ToSeverityLevel().ToSummaryHtml());

            AppendErrorSummary(html, monitor.State);

            AppendCombinedAvailabilitySummary(html, monitor.Statistics, "Summary".HtmlEncode());
            AppendAvailabilityTable(html, monitor.Statistics, "Availability".HtmlEncode());
            AppendPerformanceTable(html, monitor.Statistics, "Performance".HtmlEncode());
            AppendRecentResponses(html, monitor);

		    var subject = string.Format("ServiceMon Script {0} is {1}", monitor.Workspace.CurrentScript.Name, monitor.State.ToSeverityLevel());
            return new NotificationMessage(subject, html.ToString());
        }

        private static void AppendMonitoringPeriod(HtmlMessageBuilder html, Monitor monitor)
        {
            var scriptName = monitor.Workspace.CurrentScript.Name;
            var stats = monitor.Statistics.Combined;
            html.AddRawFormat("Script <b>{0}</b> has been running since {1:G}", scriptName.HtmlEncode(), stats.StartTime);
        }

        private static void AppendInstanceInformation(HtmlMessageBuilder html)
        {
            html.AddRawFormat("<p><i>Message sent at {0:G} from ServiceMon (v{1}), running on {2}</i></p>", DateTime.Now, ServiceMon.VersionNumber.ToString().HtmlEncode(), Dns.GetHostName().HtmlEncode());
        }

        private static void AppendRecentResponses(HtmlMessageBuilder html, Monitor monitor)
        {
            html.AddHeading(string.Format("Recent Responses (maximum of {0})", monitor.RecentResponses.MaximumItemCount).HtmlEncode());

            var bullets = monitor.RecentResponses.Items.Select(FormatResponse);

            html.AddBullets(bullets);
        }

        private static IHtmlString FormatResponse(RequestResult i)
        {
            var htmlColor = (i is RequestErrorResult) ? "red" : "black";
            return string.Format("<span style='color:{0}'>{1}</span>", htmlColor, i.ToListItemText().HtmlEncode()).AsRawHtml();
        }

        private static void AppendAvailabilityTable(HtmlMessageBuilder html, OperationStatisticsMap stats, IHtmlString heading)
        {
            html.AddHeading(heading);

            AppendStatisticsApplicableDateRange(html, stats);

            var availabilityTable = new BasicTable();
            availabilityTable.AddHeaderRow(new[] { "Operation", "Requests", "Successful", "Failed", "Success Rate" });
            foreach (var o in stats.Operations)
            {
                var successRate = o.GetSuccessRate();
                var successColor = o.SuccessCount > 0 ? HtmlColorSuccess : "#000";
                var failureColor = o.FailureCount > 0 ? HtmlColourFailure : "#000";

                availabilityTable.AddRow(new[]
                                             {
                                                 o.Description.HtmlEncode(),
                                                 o.ProcessedCount.ToString("0").HtmlEncode(),
                                                 string.Format("<span style='color:{0}'>{1:0}</span>", successColor, o.SuccessCount).AsRawHtml(),
                                                 string.Format("<span style='color:{0}'>{1:0}</span>", failureColor, o.FailureCount).AsRawHtml(),
                                                 (successRate == null ? "" : (successRate.Value * 100).ToString("N3") + "%").HtmlEncode()
                                             });
            }

            html.AddTable(availabilityTable);
        }

        private static void AppendStatisticsApplicableDateRange(HtmlMessageBuilder html, OperationStatisticsMap stats)
        {
            html.AddLine(string.Format("({0} to {1})", stats.Combined.StartTime, DateTime.Now).HtmlEncode());
        }

        private static void AppendCombinedAvailabilitySummary(HtmlMessageBuilder html, OperationStatisticsMap stats, IHtmlString heading)
        {
            html.AddHeading(heading);
            AppendStatisticsApplicableDateRange(html, stats);
            html.AddRawFormat("<b>{0}</b> Request{1} have been sent and received", stats.Combined.ProcessedCount, stats.Combined.ProcessedCount == 1 ? "" : "s");

            var successColor = stats.Combined.SuccessCount > 0 ? HtmlColorSuccess : "#000";
            var failureColor = stats.Combined.FailureCount > 0 ? HtmlColourFailure : "#000";

            html.AddBullets(new[]
                                {
                                    string.Format("<span style='color:{0}'><b>{1}</b> {2} successful</span>", successColor, stats.Combined.SuccessCount, stats.Combined.SuccessCount == 1 ? "was" : "were").AsRawHtml(),
                                    string.Format("<span style='color:{0}'><b>{1}</b> failed</span>", failureColor,stats.Combined.FailureCount).AsRawHtml()
                                });

            var successPercent = (100m*stats.Combined.SuccessCount)/stats.Combined.ProcessedCount;
            html.AddRawFormat("<p>The current success rate is <b>{0:N3}%</b></p>", successPercent);
        }

        private static void AppendPerformanceTable(HtmlMessageBuilder html, OperationStatisticsMap stats, IHtmlString heading)
        {
            html.AddHeading(heading);

            var performanceTable = new BasicTable();
            performanceTable.AddHeaderRow(new[] { "Operation", "Minimum", "Mean", "90th", "99th", "99.9th", "99.99th", "Maximum" });
            
            foreach(var o in stats.Operations)
            {
                var percentiles = o.GetResponseTimePercentiles(new[] {90m, 99m, 99.9m, 99.99m});

                var values = new List<IHtmlString>
                                 {
                                     o.Description.HtmlEncode(),
                                     GetMillisecondText(o.MinimumProcessingTime).HtmlEncode(),
                                     GetMillisecondText(o.MeanProcessingTime).HtmlEncode()
                                 };
                values.AddRange(percentiles.Select(i => GetMillisecondText(i.Value).HtmlEncode()));
                values.Add(GetMillisecondText(o.MaximumProcessingTime).HtmlEncode());
                performanceTable.AddRow(values);
            }
            
            html.AddTable(performanceTable);
        }

        private static void AppendErrorSummary(HtmlMessageBuilder html, MonitorState monitorState)
        {
            if (monitorState.HasOperationFailures)
            {
                html.AddBullets(BuildErrorList(monitorState));
            }
        }

        private static IEnumerable<IHtmlString> BuildErrorList(MonitorState monitorState)
        {
            var firstErrorHtml = monitorState.FirstOperationError.ToString().HtmlEncode();
            var latestErrorHtml = monitorState.LatestOperationError.ToString().HtmlEncode();

            if (firstErrorHtml != latestErrorHtml)
            {
                return new[]
                           {
                               string.Format("<span style='color:{0}'>First error:  {1}</span>", HtmlColourFailure, firstErrorHtml).AsRawHtml(),
                               string.Format("<span style='color:{0}'>Latest error: {1}</span>", HtmlColourFailureFresh, latestErrorHtml).AsRawHtml()
                           };
            }
            return new[]
                       {
                           string.Format("<span style='color:#D00'>Error details: {0}</span>", latestErrorHtml).AsRawHtml()
                       };
        }

        private static string GetMillisecondText(TimeSpan? value)
        {
            return value == null ? "n/a" : ((int)value.Value.TotalMilliseconds).ToString("0") + " ms";
        }
    }
}