Click here to Skip to main content
15,886,518 members
Articles / Web Development / ASP.NET

HTTP Data Client - Web Scraping

Rate me:
Please Sign up or sign in to vote.
4.79/5 (8 votes)
21 Jul 2011CPOL12 min read 47.4K   1.7K   56  
A HTTPWebRequest based library which abstracts how data is retrieved from web sources.
using System;
using System.Reflection;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Xml;
using System.Xml.XPath;
using System.Xml.Xsl;
using System.Collections.Generic;
using System.Security.Cryptography;

using HttpData.Client.MsHtmlToXml;
using HttpData.Client.Pdf;
using HtmlAgilityPack;

namespace HttpData.Client
{
    /// <summary>
    /// Provides functionality for querying and processing data from different web sources.
    /// </summary>
    public class HDPCommand : IHDPCommand
    {
        #region Private Variables

        private HDPConnection _connection;
        private HDPParameterCollection _parameters;
        private HDPCommandType _commandType;
        private string _commandText;
        private string _xsltTemplate;
        private HttpWebRequest request;
        private Cookie _ckSession;
        private string _xsltDir;
        private bool activatePool;
        private bool useMsHtml;
        private readonly XslCompiledTransform tr;
        private readonly XmlUrlResolver xmlResolver;
        private bool ignoreParamsName;
        private Parser pdfParser;
        private string commandResponse;
        private string responseUri;
        private string responsePath;
        private HttpWebResponse httpResponse;
        private string lastError;
        private long contentLength;
        #endregion

        #region Constants
        private const string XSLT_EXTENSION = ".xsl";
        #endregion

        #region Properties
        /// <summary>
        /// Get or set the command type.
        /// </summary>
        public HDPCommandType CommandType
        {
            get { return _commandType; }
            set
            {
                if ((value != HDPCommandType.Get) && (value != HDPCommandType.Post))
                    throw new ArgumentException("Invalid command type", "value");

                _commandType = value;
            }
        }

        /// <summary>
        /// Get or set the command text. 
        /// </summary>
        public string CommandText
        {
            get { return _commandText; }
            set { _commandText = value; }
        }

        /// <summary>
        /// Get the xslt template associated with this command object.
        /// </summary>
        public string XSLTTemplate
        {
            get { return _xsltTemplate; }
        }

        /// <summary>
        /// Get or set the command connection object.
        /// </summary>
        public IHDPConnection Connection
        {
            get { return _connection; }
            set { _connection = (HDPConnection)value; }
        }

        /// <summary>
        /// Get or set the command parameters collection.
        /// </summary>
        public IHDPParameterCollection Parameters
        {
            get
            {
                if (_parameters == null)
                    _parameters = new HDPParameterCollection();
                return _parameters;
            }
            set { _parameters = (HDPParameterCollection)value; }
        }

        /// <summary>
        /// Get or set the command timeout.
        /// </summary>
        public int CommandTimeout { get; set; }

        /// <summary>
        /// Get or set the xslt templates directory for this command object.
        /// </summary>
        public string XSLTDirectory
        {
            get { return _xsltDir; }
            set { _xsltDir = value; }
        }

        /// <summary>
        /// Get or set the session cookies for this command object.
        /// </summary>
        public Cookie SessionCookie
        {
            get { return _ckSession; }
            set
            {
                if (_ckSession == null)
                    _ckSession = value;
            }
        }

        /// <summary>
        /// Get or set the activation pool value used to determine if data will be retrieved from cache.
        /// </summary>
        public bool ActivatePool
        {
            get { return activatePool; }
            set { activatePool = value; }
        }

        /// <summary>
        /// Get or set if the MsHtml library should be used for Html to Xml conversion.
        /// </summary>
        public bool UseMsHtml
        {
            get { return useMsHtml; }
            set { useMsHtml = value; }
        }

        /// <summary>
        /// Get or set if parameters names should be ignored.
        /// If is set to 'true' then the data will be sent as raw.
        /// </summary>
        public bool IgnoreParamsName
        {
            get { return ignoreParamsName; }
            set { ignoreParamsName = value; }
        }

        /// <summary>
        /// Get the response retrieved from the server.
        /// </summary>
        public string Response
        {
            get { return commandResponse; }
        }

        /// <summary>
        /// Get web resource URI.
        /// </summary>
        public string Uri
        {
            get { return responseUri; }
        }

        /// <summary>
        /// Get web resource absolute path.
        /// </summary>
        public string Path
        {
            get { return responsePath; }
        }

        /// <summary>
        /// Get the last error occurend.
        /// </summary>
        public string LastError { get { return lastError; } }

        /// <summary>
        /// Get the content length of response.
        /// </summary>
        public long ContentLength { get { return contentLength; } }
        #endregion

        #region .ctor
        /// <summary>
        /// Instantiate a new HDPCommand object.
        /// </summary>
        public HDPCommand()
        {
            CommandTimeout = 1000;
            _commandType = HDPCommandType.Get;

            tr = new XslCompiledTransform();
            xmlResolver = new XmlUrlResolver();

            pdfParser = new Parser();
            ServicePointManager.DefaultConnectionLimit = 150;
        }

        /// <summary>
        /// Instantiate a new HDPCommand object.
        /// </summary>
        /// <param name="connection">Set HDPConnection object to current command object.</param>
        public HDPCommand(HDPConnection connection)
        {
            _connection = connection;
            CommandTimeout = 1000;

            tr = new XslCompiledTransform();
            xmlResolver = new XmlUrlResolver();

            pdfParser = new Parser();
            ServicePointManager.DefaultConnectionLimit = 150;
        }

        /// <summary>
        /// Instantiate a new HDPCommand object.
        /// </summary>
        /// <param name="connection">Set HDPConnection object to current command object.</param>
        /// <param name="commandType">Set HDPCommandType for the current command object.</param>
        public HDPCommand(HDPConnection connection, HDPCommandType commandType)
        {
            _commandType = commandType;
            _connection = connection;
            CommandTimeout = 1000;

            tr = new XslCompiledTransform();
            xmlResolver = new XmlUrlResolver();

            pdfParser = new Parser();
            ServicePointManager.DefaultConnectionLimit = 150;
        }

        /// <summary>
        /// Instantiate a new HDPCommand object.
        /// </summary>
        /// <param name="connection">Set HDPConnection object to current command object.</param>
        /// <param name="commandType">Set HDPCommandType for the current command object.</param>
        /// <param name="commandText">Set the command text for the current command object.</param>
        public HDPCommand(HDPConnection connection, HDPCommandType commandType, string commandText)
        {
            _commandType = commandType;
            _connection = connection;
            _commandText = commandText;
            CommandTimeout = 1000;

            tr = new XslCompiledTransform();
            xmlResolver = new XmlUrlResolver();

            pdfParser = new Parser();
            ServicePointManager.DefaultConnectionLimit = 150;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="connection">Set HDPConnection object to current command object.</param>
        /// <param name="commandText">Set the command text for the current command object.</param>
        public HDPCommand(HDPConnection connection, string commandText)
        {
            _connection = connection;
            _commandText = commandText;
            CommandTimeout = 1000;

            tr = new XslCompiledTransform();
            xmlResolver = new XmlUrlResolver();

            pdfParser = new Parser();
            ServicePointManager.DefaultConnectionLimit = 150;
        }
        #endregion

        #region Public Methods
        /// <summary>
        /// Create a new IHDPParameter object.
        /// </summary>
        /// <returns>IHDPParameter parameter object.</returns>
        public IHDPParameter CreateParameter()
        {
            return new HDPParameter();
        }

        /// <summary>
        /// Get the parameters number.
        /// </summary>
        /// <returns>Number of parameters.</returns>
        public int GetParametersCount()
        {
            if (_parameters == null)
                return 0;

            return _parameters.Count;
        }

        /// <summary>
        /// Execute a query against the web server and does not read the response stream.
        /// </summary>
        /// <returns>True is the command executed with success otherwise false.</returns>
        public bool Execute()
        {
            try
            {
                PrepareRequest();
                httpResponse = (HttpWebResponse)request.GetResponse();

                responseUri = httpResponse.ResponseUri.AbsoluteUri;
                responsePath = httpResponse.ResponseUri.AbsolutePath;

                foreach (Cookie cookie in httpResponse.Cookies)
                {
                    if (_connection.Cookies.Count == 20)
                        _connection.Cookies = new CookieCollection();

                    if (cookie.Value.Length <= 4096)
                        _connection.Cookies.Add(cookie);
                }

                if (httpResponse.StatusCode == HttpStatusCode.Found || httpResponse.StatusCode == HttpStatusCode.SeeOther)
                {
                    string redirectURL = httpResponse.Headers["Location"];
                    httpResponse.Close();

                    _commandType = HDPCommandType.Get;
                    _connection.Open(redirectURL);

                    return Execute();
                }

                if (httpResponse.StatusCode == HttpStatusCode.OK)
                {
                    contentLength = httpResponse.ContentLength;
                    httpResponse.Close();
                    return true;
                }

                return false;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return false;
        }

        /// <summary>
        /// Execute a query against the web server.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>True is the command executed with success otherwise false.</returns>
        public bool Execute(bool clearParams)
        {
            try
            {
                PrepareRequest();
                httpResponse = (HttpWebResponse)request.GetResponse();

                responseUri = httpResponse.ResponseUri.AbsoluteUri;
                responsePath = httpResponse.ResponseUri.AbsolutePath;

                foreach (Cookie cookie in httpResponse.Cookies)
                {
                    if (_connection.Cookies.Count == 20)
                        _connection.Cookies = new CookieCollection();

                    if (cookie.Value.Length <= 4096)
                        _connection.Cookies.Add(cookie);
                }

                if (httpResponse.StatusCode == HttpStatusCode.Found || httpResponse.StatusCode == HttpStatusCode.SeeOther)
                {
                    string redirectURL = httpResponse.Headers["Location"];
                    httpResponse.Close();

                    _commandType = HDPCommandType.Get;
                    _connection.Open(redirectURL);

                    return Execute(clearParams);
                }

                if (httpResponse.StatusCode == HttpStatusCode.OK)
                {
                    contentLength = httpResponse.ContentLength;

                    Stream responseStream = httpResponse.GetResponseStream();
                    StreamReader sr = new StreamReader(responseStream, Encoding.UTF8);
                    commandResponse = sr.ReadToEnd();

                    responseStream.Flush();
                    responseStream.Close();

                    sr.Close();
                    httpResponse.Close();
                }


                if (clearParams)
                    ClearParameters();
                if (httpResponse.StatusCode == HttpStatusCode.OK)
                    return true;

                return false;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return false;
        }

        /// <summary>
        /// Execute a query against the web server.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>Returns the underlying http response stream.</returns>
        public Stream ExecuteStream(bool clearParams)
        {
            try
            {
                PrepareRequest();
                httpResponse = (HttpWebResponse)request.GetResponse();
                contentLength = httpResponse.ContentLength;

                responseUri = httpResponse.ResponseUri.AbsoluteUri;
                responsePath = httpResponse.ResponseUri.AbsolutePath;

                Stream responseStream = httpResponse.GetResponseStream();
                return responseStream;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return null;
        }

        /// <summary>
        /// Closes the http response object..
        /// Usable only with ExecuteStream method.  
        /// </summary>
        public void CloseResponse()
        {
            try
            {
                httpResponse.Close();
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
            }
        }

        /// <summary>
        ///  Execute a query against the web server.
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>Boolean value which if 'true' indicates if the query executed with success, otherwise 'false'.</returns>
        public bool Execute(IHDPTemplate template, string member, bool clearParams)
        {
            XPathNavigator xpNav = ExecuteDocument(clearParams).CreateNavigator();
            template.XPathDataSource = xpNav;

            bool result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = (bool)method.Invoke(template, null);
            }
            catch (Exception ex)
            {
                ExceptionHelper.DebugException(ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL), ex);
                result = false;
            }

            return result;
        }

        /// <summary>
        /// Execute a expression against the web server and return the number of results.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>Number of results determined by the expression.</returns>
        public int ExecuteNonQuery(bool clearParams)
        {
            try
            {
                if (!string.IsNullOrEmpty(_commandText))
                {
                    XPathNavigator _xpNav = ExecuteNavigator(clearParams);
                    XPathExpression _xpExpresion = _xpNav.Compile(_commandText);
                    XPathNodeIterator _xpNIterator = _xpNav.Select(_xpExpresion);

                    return _xpNIterator.Count;
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return -1;
        }

        /// <summary>
        /// Execute a query against the web server and return a XPathNavigator object used to navige thru the query result.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>XPathNavigator object used to navige thru the query result.</returns>
        public XPathNavigator ExecuteNavigator(bool clearParams)
        {
            XPathNavigator _xpNav = null;
            string guid = CreateDataSourceId();
            string xmltext = null;
            string _sResult = null;

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    xmltext = obj != null ? (string)obj.Value : null;
                }

                if (xmltext == null)
                {
                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    foreach (Cookie cookie in httpResponse.Cookies)
                    {
                        if (_connection.Cookies.Count == 20)
                            _connection.Cookies = new CookieCollection();

                        if (cookie.Value.Length <= 4096)
                            _connection.Cookies.Add(cookie);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.Found || httpResponse.StatusCode == HttpStatusCode.SeeOther)
                    {
                        string redirectURL = httpResponse.Headers["Location"];
                        httpResponse.Close();

                        _commandType = HDPCommandType.Get;
                        _connection.Open(redirectURL);

                        return ExecuteNavigator(clearParams);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.OK)
                    {
                        contentLength = httpResponse.ContentLength;

                        Stream responseStream = httpResponse.GetResponseStream();
                        StreamReader sr = new StreamReader(responseStream, Encoding.UTF8);
                        string strResponse = commandResponse = sr.ReadToEnd();

                        responseStream.Flush();
                        responseStream.Close();

                        sr.Close();
                        httpResponse.Close();

                        int index = strResponse.IndexOf("<html");
                        if (index == -1)
                        {
                            strResponse = strResponse.Insert(0, "<html>");
                            strResponse = strResponse + "</html>";
                        }
                        else
                        {
                            int lindex = strResponse.IndexOf(">", index);
                            strResponse = strResponse.Remove(0, lindex + 1);
                            strResponse = strResponse.Insert(0, "<html>");
                        }

                        if (useMsHtml)
                        {
                            AutoResetEvent resultEvent = new AutoResetEvent(false);
                            WebHost host = new WebHost(strResponse, true, HtmlFixOption.RemoveScripts | HtmlFixOption.RemoveComments,
                                delegate(string value) { _sResult = value; }, resultEvent);
                            WaitHandle.WaitAll(new[] { resultEvent });

                            host.Dispose();
                        }
                        else
                        {
                            MemoryStream memoryStream = new MemoryStream();

                            HtmlDocument doc = new HtmlDocument { OptionOutputAsXml = true };
                            doc.LoadHtml(strResponse);
                            doc.Save(memoryStream);

                            memoryStream.Flush();
                            memoryStream.Position = 0;

                            StreamReader streamReader = new StreamReader(memoryStream);
                            _sResult = streamReader.ReadToEnd();

                            memoryStream.Close();
                            streamReader.Close();
                        }

                        XPathDocument _xpResult = new XPathDocument(new StringReader(_sResult));
                        _xpNav = _xpResult.CreateNavigator();

                        if (activatePool && _connection.UseCahe && _connection.Cache != null)
                        {
                            HDPCacheObject obj = new HDPCacheObject(guid, _xpNav.OuterXml);
                            _connection.Cache.AddObject(obj);
                        }
                    }
                }
                else
                {
                    XPathDocument _xpResult = new XPathDocument(new StringReader(xmltext));
                    _xpNav = _xpResult.CreateNavigator();
                }

                if (clearParams)
                    ClearParameters();
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xpNav = new XmlDocument().CreateNavigator();
            }

            return _xpNav;
        }

        /// <summary>
        /// Execute a query against the web server and return a IXPathNavigable object used to navige thru the query result.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navige thru the query result.</returns>
        public IXPathNavigable ExecuteDocument(bool clearParams)
        {
            XmlDocument _xmlResult = null;
            string guid = CreateDataSourceId();
            string _sResult = null;

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    if (obj != null)
                    {
                        _xmlResult = new XmlDocument();
                        _xmlResult.LoadXml((string)obj.Value);
                    }
                }

                if (_xmlResult == null)
                {
                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    foreach (Cookie cookie in httpResponse.Cookies)
                    {
                        if (_connection.Cookies.Count == 20)
                            _connection.Cookies = new CookieCollection();

                        if (cookie.Value.Length <= 4096)
                            _connection.Cookies.Add(cookie);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.Found || httpResponse.StatusCode == HttpStatusCode.SeeOther)
                    {
                        string redirectURL = httpResponse.Headers["Location"];
                        httpResponse.Close();

                        _commandType = HDPCommandType.Get;
                        _connection.Open(redirectURL);

                        return ExecuteDocument(clearParams);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.OK)
                    {
                        contentLength = httpResponse.ContentLength;

                        Stream responseStream = httpResponse.GetResponseStream();
                        StreamReader sr = new StreamReader(responseStream, Encoding.UTF8);
                        string strResponse = commandResponse = sr.ReadToEnd();

                        responseStream.Flush();
                        responseStream.Close();

                        sr.Close();
                        httpResponse.Close();

                        int index = strResponse.IndexOf("<html");
                        if (index == -1)
                        {
                            strResponse = strResponse.Insert(0, "<html>");
                            strResponse = strResponse + "</html>";
                        }
                        else
                        {
                            int lindex = strResponse.IndexOf(">", index);
                            strResponse = strResponse.Remove(0, lindex + 1);
                            strResponse = strResponse.Insert(0, "<html>");
                        }

                        if (useMsHtml)
                        {
                            AutoResetEvent resultEvent = new AutoResetEvent(false);
                            WebHost host = new WebHost(strResponse, true, HtmlFixOption.RemoveScripts | HtmlFixOption.RemoveComments,
                                delegate(string value) { _sResult = value; }, resultEvent);
                            WaitHandle.WaitAll(new[] { resultEvent });

                            host.Dispose();
                        }
                        else
                        {
                            MemoryStream memoryStream = new MemoryStream();

                            HtmlDocument doc = new HtmlDocument { OptionOutputAsXml = true };
                            doc.LoadHtml(strResponse);
                            doc.Save(memoryStream);

                            memoryStream.Flush();
                            memoryStream.Position = 0;

                            StreamReader streamReader = new StreamReader(memoryStream);
                            _sResult = streamReader.ReadToEnd();

                            memoryStream.Close();
                            streamReader.Close();
                        }

                        _xmlResult = new XmlDocument();
                        _xmlResult.Load(new StringReader(_sResult));

                        if (activatePool && _connection.UseCahe && _connection.Cache != null)
                        {
                            HDPCacheObject obj = new HDPCacheObject(guid, _sResult);
                            _connection.Cache.AddObject(obj);
                        }
                    }

                    if (clearParams)
                        ClearParameters();
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xmlResult = new XmlDocument();
            }

            return _xmlResult;
        }

        /// <summary>
        /// Execute a query against the web server and return a byte[] object which contains the binary query result. Used when querying binary content from web server (E.g: PDF files).
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>Byte array object which contains the binary query result.</returns>
        public byte[] ExecuteBinary(bool clearParams)
        {
            byte[] buffer = null;
            string guid = CreateDataSourceId();

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    buffer = obj != null ? (byte[])obj.Value : null;
                }

                if (buffer == null || buffer.Length == 0)
                {
                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();
                    contentLength = httpResponse.ContentLength;

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    Stream responseStream = httpResponse.GetResponseStream();

                    buffer = ReadFully(responseStream, (int)httpResponse.ContentLength, false);
                    httpResponse.Close();
                    responseStream.Flush();
                    responseStream.Close();

                    if (activatePool && _connection.UseCahe && _connection.Cache != null)
                    {
                        HDPCacheObject obj = new HDPCacheObject(guid, buffer);
                        _connection.Cache.AddObject(obj);
                    }
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                buffer = new byte[] { };
            }

            return buffer;
        }

        /// <summary>
        /// Execute a query against the web server and return a byte[] object which contains the binary query result. Used when querying binary content from web server (E.g: PDF files).
        /// </summary>
        /// <param name="boundaryLimit">Specify the limit of the buffer which must be read.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>Byte array object which contains the binary query result.</returns>
        public byte[] ExecuteBinary(int boundaryLimit, bool clearParams)
        {
            byte[] buffer = null;
            string guid = CreateDataSourceId();

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    buffer = obj != null ? (byte[])obj.Value : null;
                }

                if (buffer == null || buffer.Length == 0)
                {
                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();
                    contentLength = httpResponse.ContentLength;

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    Stream responseStream = httpResponse.GetResponseStream();

                    buffer = ReadFully(responseStream, boundaryLimit, true);
                    httpResponse.Close();
                    responseStream.Flush();
                    responseStream.Close();

                    if (activatePool && _connection.UseCahe && _connection.Cache != null)
                    {
                        HDPCacheObject obj = new HDPCacheObject(guid, buffer);
                        _connection.Cache.AddObject(obj);
                    }
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                buffer = new byte[] { };
            }

            return buffer;
        }

        /// <summary>
        ///  Execute a query against the web server and return a string object which contains the representation of the binary query result. Used when querying binary content from web server (E.g: PDF files).
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the binary query result.</returns>
        public string ExecuteBinaryConversion(bool clearParams)
        {
            string strResponse = null;
            string guid = CreateDataSourceId();

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    strResponse = obj != null ? (string)obj.Value : null;
                }

                if (strResponse == null && _connection.Cache != null)
                {
                    string content = "";

                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();
                    contentLength = httpResponse.ContentLength;

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    Stream responseStream = httpResponse.GetResponseStream();

                    byte[] buffer = ReadFully(responseStream, (int)httpResponse.ContentLength, false);
                    httpResponse.Close();
                    responseStream.Flush();
                    responseStream.Close();

                    if (httpResponse.ContentType == HDPContentType.AP_PDF && buffer != null)
                    {
                        string fileName = UrlToPath(httpResponse.ResponseUri.OriginalString);
                        string filePath = _connection.Cache.WriteToDisk(buffer, fileName);

                        MemoryStream ms = new MemoryStream(buffer);

                        strResponse = content = pdfParser.ExtractText(filePath);

                        ms.Close();
                    }

                    if (activatePool && _connection.UseCahe && _connection.Cache != null)
                    {
                        if (content != null)
                        {
                            HDPCacheObject obj = new HDPCacheObject(guid, content);
                            _connection.Cache.AddObject(obj);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                strResponse = string.Empty;
            }

            return strResponse;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a string object which contains the representation of the binary query result. Used when querying binary content from web server (E.g: PDF files).
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the binary query result.</returns>
        public string ExecuteBinary(IHDPTemplate template, string member, bool clearParams)
        {
            byte[] buffer = ExecuteBinary(clearParams);
            template.BinaryDataSource = buffer;

            string result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = (string)method.Invoke(template, null);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                result = string.Empty;
            }

            return result;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a string object which contains the representation of the binary query result. Used when querying binary content from web server (E.g: PDF files).
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the binary query result.</returns>
        public string ExecuteBinaryConversion(IHDPTemplate template, string member, bool clearParams)
        {
            string content = ExecuteBinaryConversion(clearParams);
            template.StringDataSource = content;

            string result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = (string)method.Invoke(template, null);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                result = string.Empty;
            }

            return result;
        }

        /// <summary>
        /// Execute a query against the web server and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(bool clearParams)
        {
            XmlDocument _xmlResult = null;
            string guid = CreateDataSourceId();
            string _sResult = null;

            try
            {
                if (activatePool && _connection.UseCahe && _connection.Cache != null)
                {
                    HDPCacheObject obj = _connection.Cache.RetrieveObject(guid);
                    if (obj != null)
                    {
                        _xmlResult = new XmlDocument();
                        _xmlResult.LoadXml((string)obj.Value);
                    }
                }

                if (_xmlResult == null)
                {
                    PrepareRequest();
                    httpResponse = (HttpWebResponse)request.GetResponse();

                    responseUri = httpResponse.ResponseUri.AbsoluteUri;
                    responsePath = httpResponse.ResponseUri.AbsolutePath;

                    foreach (Cookie cookie in httpResponse.Cookies)
                    {
                        if (_connection.Cookies.Count == 20)
                            _connection.Cookies = new CookieCollection();

                        if (cookie.Value.Length <= 4096)
                            _connection.Cookies.Add(cookie);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.Found || httpResponse.StatusCode == HttpStatusCode.SeeOther)
                    {
                        string redirectURL = httpResponse.Headers["Location"];
                        httpResponse.Close();

                        _commandType = HDPCommandType.Get;
                        _connection.Open(redirectURL);

                        return ExecuteString(clearParams);
                    }

                    if (httpResponse.StatusCode == HttpStatusCode.OK)
                    {
                        contentLength = httpResponse.ContentLength;

                        Stream responseStream = httpResponse.GetResponseStream();
                        StreamReader sr = new StreamReader(responseStream, Encoding.UTF8);
                        string strResponse = commandResponse = sr.ReadToEnd();

                        responseStream.Flush();
                        responseStream.Close();

                        sr.Close();
                        httpResponse.Close();

                        if (strResponse.IndexOf("<html") == -1)
                        {
                            strResponse = strResponse.Insert(0, "<html>");
                            strResponse = strResponse + "</html>";
                        }

                        if (useMsHtml)
                        {
                            AutoResetEvent resultEvent = new AutoResetEvent(false);
                            WebHost host = new WebHost(strResponse, true, HtmlFixOption.RemoveScripts | HtmlFixOption.RemoveComments,
                                delegate(string value) { _sResult = value; }, resultEvent);
                            WaitHandle.WaitAll(new[] { resultEvent });

                            host.Dispose();
                        }
                        else
                        {
                            MemoryStream memoryStream = new MemoryStream();

                            HtmlDocument doc = new HtmlDocument { OptionOutputAsXml = true };
                            doc.LoadHtml(strResponse);
                            doc.Save(memoryStream);

                            memoryStream.Flush();
                            memoryStream.Position = 0;

                            StreamReader streamReader = new StreamReader(memoryStream);
                            _sResult = streamReader.ReadToEnd();

                            memoryStream.Close();
                            streamReader.Close();
                        }

                        _xmlResult = new XmlDocument();
                        _xmlResult.Load(new StringReader(_sResult));

                        if (activatePool && _connection.UseCahe && _connection.Cache != null)
                        {
                            HDPCacheObject obj = new HDPCacheObject(guid, _sResult);
                            _connection.Cache.AddObject(obj);
                        }
                    }
                    if (clearParams)
                        ClearParameters();
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xmlResult = null;
            }

            return _xmlResult != null ? _xmlResult.InnerXml : "";
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a  xslt template and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="stylesheet">Name of xslt template.</param>
        /// <param name="args">Arguments for the xslt template.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(string stylesheet, XsltArgumentList args, bool clearParams)
        {
            _xsltTemplate = stylesheet;
            XmlDocument _xmlResult;

            try
            {
                XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
                string _stylesheet = _xsltDir + stylesheet + XSLT_EXTENSION;
                StringWriter sw = new StringWriter();

                tr.Load(_stylesheet, XsltSettings.TrustedXslt, xmlResolver);
                tr.Transform(xdNav, args, sw);

                if (sw.ToString().Length > 0)
                {
                    XmlTextReader xmlReader = new XmlTextReader(new StringReader(sw.ToString()));
                    _xmlResult = new XmlDocument();
                    _xmlResult.Load(xmlReader);

                    if (clearParams)
                        ClearParameters();
                    return _xmlResult.InnerXml;
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return string.Empty;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a xpath expression and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">XPath expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(string expression, bool clearParams)
        {
            XPathNodeIterator xniNode;

            try
            {
                XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
                xniNode = xdNav.Select(expression);
                xniNode.MoveNext();
            }
            catch (Exception ex)
            {
                ExceptionHelper.DebugException(ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL), ex);
                xniNode = null;
            }

            return xniNode != null ? xniNode.Current.InnerXml : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of xpath expression and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(Dictionary<string, string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            StringBuilder sbResult = new StringBuilder();
            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry.Value);
                    xniNode.MoveNext();
                    sbResult.Append(xniNode.Current.InnerXml);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of xpath expression and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(List<string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (string entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry);
                    xniNode.MoveNext();
                    sbResult.Append(xniNode.Current.InnerXml);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a regular expression and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">Regular expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(string expression, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            StringBuilder sbResult = new StringBuilder();

            try
            {
                string doc = xmlDoc.OuterXml;
                Regex regEx = new Regex(expression, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                Match match = regEx.Match(doc);

                foreach (Capture capture in match.Captures)
                {
                    sbResult.Append(capture.Value);
                    sbResult.Append(" ");
                }

            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of regular expressions and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(Dictionary<string, string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    Regex regEx = new Regex(entry.Value, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);

                    foreach (Capture capture in match.Captures)
                    {
                        sbResult.Append(capture.Value);
                        sbResult.Append(" ");
                    }
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(List<string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (string entry in expressions)
                {
                    Regex regEx = new Regex(entry, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);

                    foreach (Capture capture in match.Captures)
                    {
                        sbResult.Append(capture.Value);
                        sbResult.Append(" ");
                    }
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a string object which contains the representation of the query result.
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result.</returns>
        public string ExecuteString(IHDPTemplate template, string member, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            template.XPathDataSource = xdNav;
            string result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = (string)method.Invoke(template, null);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                result = string.Empty;
            }

            return result;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a xpath expression and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expression">XPath expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(string expression, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            XPathNodeIterator xniNode;

            try
            {
                xniNode = xdNav.Select(expression);
                xniNode.MoveNext();
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                xniNode = null;
            }

            return xniNode != null ? xniNode.Current.Value : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of xpath expression and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expressions">Dictionary of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(Dictionary<string, string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry.Value);
                    xniNode.MoveNext();

                    sbResult.Append(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of xpath expression and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expressions">List of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(List<string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (string entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry);
                    xniNode.MoveNext();

                    sbResult.Append(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a regular expression and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expression">Regular expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(string expression, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            Match match;

            try
            {
                Regex regEx = new Regex(expression, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                match = regEx.Match(doc);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                match = null;
            }

            return match != null ? match.Value : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(Dictionary<string, string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    Regex regEx = new Regex(entry.Value, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);
                    sbResult.Append(match.Value);
                    sbResult.Append(" ");
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a string object which contains the representation of the query result value.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String object which contains the representation of the query result value.</returns>
        public string ExecuteValue(List<string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            StringBuilder sbResult = new StringBuilder();

            try
            {
                foreach (string entry in expressions)
                {
                    Regex regEx = new Regex(entry, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);
                    sbResult.Append(match.Value);
                    sbResult.Append(" ");
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return sbResult.ToString();
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a xpath expression and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="expression">XPath expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(string expression, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            XPathNodeIterator xniNode = xdNav.Select(expression);

            XmlDocument xmlDoc = new XmlDocument();
            XmlNode root = xmlDoc.CreateNode(XmlNodeType.Element, "table", "");
            XmlNode row = xmlDoc.CreateNode(XmlNodeType.Element, "row", "");

            try
            {
                StringBuilder sbResult = new StringBuilder();
                while (xniNode.MoveNext())
                    sbResult.Append(xniNode.Current.OuterXml);

                XmlNode node = xmlDoc.CreateElement("column");
                node.InnerXml = sbResult.ToString();
                row.AppendChild(node);
                root.AppendChild(row);
                xmlDoc.AppendChild(root);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return xmlDoc;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of xpath expression and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="expressions">Dictionary of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(Dictionary<string, string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            XmlDocument xmlDoc = new XmlDocument();
            XmlNode root = xmlDoc.CreateNode(XmlNodeType.Element, "table", "");
            XmlNode row = xmlDoc.CreateNode(XmlNodeType.Element, "row", "");

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry.Value);
                    StringBuilder sbResult = new StringBuilder();
                    while (xniNode.MoveNext())
                        sbResult.Append(xniNode.Current.OuterXml);

                    XmlNode node = xmlDoc.CreateElement(entry.Key);
                    node.InnerXml = sbResult.ToString();
                    row.AppendChild(node);
                }

                root.AppendChild(row);
                xmlDoc.AppendChild(root);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return xmlDoc;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of xpath expression and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="expressions">List of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(List<string> expressions, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            XmlDocument xmlDoc = new XmlDocument();
            XmlNode root = xmlDoc.CreateNode(XmlNodeType.Element, "table", "");
            XmlNode row = xmlDoc.CreateNode(XmlNodeType.Element, "row", "");
            int index = 0;

            try
            {
                foreach (string entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry);

                    StringBuilder sbResult = new StringBuilder();
                    while (xniNode.MoveNext())
                        sbResult.Append(xniNode.Current.OuterXml);

                    XmlNode node = xmlDoc.CreateElement("Column_" + index);
                    node.InnerXml = sbResult.ToString();
                    row.AppendChild(node);

                    index++;
                }

                root.AppendChild(row);
                xmlDoc.AppendChild(root);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return xmlDoc;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of regular expressions and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="expressions">Dictionary of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(Dictionary<string, string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;

            XmlDocument xmlDocOut = new XmlDocument();
            XmlNode root = xmlDocOut.CreateNode(XmlNodeType.Element, "table", "");
            XmlNode row = xmlDoc.CreateNode(XmlNodeType.Element, "row", "");

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    Regex regEx = new Regex(entry.Value, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);

                    if (match.Success)
                    {
                        XmlNode node = xmlDocOut.CreateElement(entry.Key);
                        node.InnerXml = match.Value;
                        row.AppendChild(node);
                    }
                }

                root.AppendChild(row);
                xmlDocOut.AppendChild(root);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return xmlDocOut;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(List<string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;

            XmlDocument xmlDocOut = new XmlDocument();
            XmlNode root = xmlDocOut.CreateNode(XmlNodeType.Element, "table", "");
            XmlNode row = xmlDoc.CreateNode(XmlNodeType.Element, "row", "");
            int index = 0;

            try
            {
                foreach (string entry in expressions)
                {
                    Regex regEx = new Regex(entry, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    Match match = regEx.Match(doc);

                    if (match.Success)
                    {
                        XmlNode node = xmlDocOut.CreateElement("Column_" + index);
                        node.InnerXml = match.Value;
                        row.AppendChild(node);
                    }

                    index++;
                }

                root.AppendChild(row);
                xmlDocOut.AppendChild(root);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);
            }

            return xmlDocOut;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a  xslt template and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="stylesheet">Name of xslt template.</param>
        /// <param name="args">Arguments for the xslt template.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(string stylesheet, XsltArgumentList args, bool clearParams)
        {
            _xsltTemplate = stylesheet;
            XmlDocument _xmlResult = null;

            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            string _stylesheet = _xsltDir + stylesheet + XSLT_EXTENSION;
            StringWriter sw = new StringWriter();

            try
            {
                tr.Load(_stylesheet, XsltSettings.TrustedXslt, xmlResolver);
                tr.Transform(xdNav, args, sw);

                if (sw.ToString().Length > 0)
                {
                    XmlTextReader xmlReader = new XmlTextReader(new StringReader(sw.ToString()));
                    _xmlResult = new XmlDocument();
                    _xmlResult.Load(xmlReader);

                    if (clearParams)
                        ClearParameters();
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xmlResult = null;
            }

            return _xmlResult;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a IXPathNavigable object used to navigate thru query result.
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>IXPathNavigable object used to navigate thru query result.</returns>
        public IXPathNavigable ExecuteDocument(IHDPTemplate template, string member, bool clearParams)
        {
            XmlDocument _xmlResult = null;
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            template.XPathDataSource = xdNav;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                string result = method.Invoke(template, null).ToString();

                if (result.Length > 0)
                {
                    XmlTextReader xmlReader = new XmlTextReader(new StringReader(result));
                    _xmlResult = new XmlDocument();
                    _xmlResult.Load(xmlReader);

                    if (clearParams)
                        ClearParameters();
                    return _xmlResult;
                }

            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xmlResult = null;
            }

            return _xmlResult;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a  xslt template and return a XPathDocument object used to navigate thru query result.
        /// </summary>
        /// <param name="stylesheet">Name of xslt template.</param>
        /// <param name="args">Arguments for the xslt template.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>XPathDocument object used to navigate thru query result.</returns>
        public XPathDocument ExecuteXPathDocument(string stylesheet, XsltArgumentList args, bool clearParams)
        {
            XPathDocument _xpResult;
            _xsltTemplate = stylesheet;

            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            string _stylesheet = _xsltDir + stylesheet + XSLT_EXTENSION;

            try
            {
                StringWriter sw = new StringWriter();
                XmlTextWriter xmlWriter = new XmlTextWriter(sw) { Formatting = Formatting.Indented };

                tr.Load(_stylesheet, XsltSettings.TrustedXslt, xmlResolver);
                tr.Transform(xdNav, args, xmlWriter);

                _xpResult = new XPathDocument(new StringReader(sw.ToString()));
                if (clearParams)
                    ClearParameters();
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                _xpResult = null;
            }

            return _xpResult;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a regular expression and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">Regular expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(string expression, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList = new List<string>();

            try
            {
                Regex regEx = new Regex(expression, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                MatchCollection matches = regEx.Matches(doc);

                foreach (Match match in matches)
                    arrList.Add(match.Value);

                if (arrList.Count > 0)
                    return arrList;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of regular expressions and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(Dictionary<string, string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList = new List<string>();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    Regex regEx = new Regex(entry.Value, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    MatchCollection matches = regEx.Matches(doc);

                    foreach (Match match in matches)
                        arrList.Add(match.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(List<string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList = new List<string>();

            try
            {
                foreach (string entry in expressions)
                {
                    Regex regEx = new Regex(entry, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    MatchCollection matches = regEx.Matches(doc);

                    foreach (Match match in matches)
                        arrList.Add(match.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a xpath expression and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">XPath expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(string expression, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                XPathNodeIterator xniNode = xdNav.Select(expression);

                while (xniNode.MoveNext())
                    arrList.Add(xniNode.Current.Value);

                return arrList;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of xpath expression and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(Dictionary<string, string> expressions, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry.Value);

                    while (xniNode.MoveNext())
                        arrList.Add(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of xpath expression and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(List<string> expressions, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                foreach (string entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry);

                    while (xniNode.MoveNext())
                        arrList.Add(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a  xslt template and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="stylesheet">Name of xslt template.</param>
        /// <param name="args">Arguments for the xslt template.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(string stylesheet, XsltArgumentList args, bool clearParams)
        {
            _xsltTemplate = stylesheet;

            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            string _stylesheet = _xsltDir + stylesheet + XSLT_EXTENSION;

            try
            {
                StringWriter sw = new StringWriter();
                XmlTextWriter xmlWriter = new XmlTextWriter(sw) { Formatting = Formatting.Indented };

                tr.Load(_stylesheet, XsltSettings.TrustedXslt, xmlResolver);
                tr.Transform(xdNav, args, xmlWriter);

                XPathDocument _xpResult = new XPathDocument(new StringReader(sw.ToString()));
                xdNav = _xpResult.CreateNavigator();

                while (xdNav.MoveToNext())
                    arrList.Add(xdNav.Value);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a List object which contains the representation of the query result.
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>List object which contains the representation of the query result.</returns>
        public List<string> ExecuteCollection(IHDPTemplate template, string member, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            template.XPathDataSource = xdNav;
            List<string> result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = method.Invoke(template, null) as List<string>;
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                result = null;
            }

            return result;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a regular expression and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">Regular expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(string expression, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList;

            try
            {
                Regex regEx = new Regex(expression, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                MatchCollection matches = regEx.Matches(doc);

                arrList = new List<string>();
                foreach (Match match in matches)
                    arrList.Add(match.Value);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of regular expressions and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(Dictionary<string, string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList = new List<string>();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    Regex regEx = new Regex(entry.Value, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    MatchCollection matches = regEx.Matches(doc);

                    foreach (Match match in matches)
                        arrList.Add(match.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of regular expressions and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of regular expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the command is executed</param>
        /// <param name="isRegEx">Specify the a regular expression is used, it must always be to true.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(List<string> expressions, bool clearParams, bool isRegEx)
        {
            XmlDocument xmlDoc = (XmlDocument)ExecuteDocument(clearParams);
            string doc = xmlDoc.OuterXml;
            List<string> arrList = new List<string>();

            try
            {
                foreach (string entry in expressions)
                {
                    Regex regEx = new Regex(entry, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.IgnorePatternWhitespace);
                    MatchCollection matches = regEx.Matches(doc);

                    foreach (Match match in matches)
                        arrList.Add(match.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a xpath expression and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expression">XPath expression.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(string expression, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                XPathNodeIterator xniNode = xdNav.Select(expression);

                while (xniNode.MoveNext())
                    arrList.Add(xniNode.Current.Value);

            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a dictionary of xpath expression and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">Dictionary of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(Dictionary<string, string> expressions, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                foreach (KeyValuePair<string, string> entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry.Value);

                    while (xniNode.MoveNext())
                        arrList.Add(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a list of xpath expression and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="expressions">List of xpath expressions.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(List<string> expressions, bool clearParams)
        {
            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();

            try
            {
                foreach (string entry in expressions)
                {
                    XPathNodeIterator xniNode = xdNav.Select(entry);

                    while (xniNode.MoveNext())
                        arrList.Add(xniNode.Current.Value);
                }
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a  xslt template and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="stylesheet">Name of xslt template.</param>
        /// <param name="args">Arguments for the xslt template.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(string stylesheet, XsltArgumentList args, bool clearParams)
        {
            _xsltTemplate = stylesheet;

            List<string> arrList = new List<string>();
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            string _stylesheet = _xsltDir + stylesheet + XSLT_EXTENSION;

            try
            {
                StringWriter sw = new StringWriter();
                XmlTextWriter xmlWriter = new XmlTextWriter(sw) { Formatting = Formatting.Indented };

                tr.Load(_stylesheet, XsltSettings.TrustedXslt, xmlResolver);
                tr.Transform(xdNav, args, xmlWriter);

                XPathDocument _xpResult = new XPathDocument(new StringReader(sw.ToString()));
                xdNav = _xpResult.CreateNavigator();

                while (xdNav.MoveToNext())
                    arrList.Add(xdNav.Value);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                arrList = null;
            }

            return arrList != null ? arrList.ToArray() : null;
        }

        /// <summary>
        /// Execute a query against the web server, on query reult it will apply a method of IHDPTemplate object and return a string array object which contains the representation of the query result.
        /// </summary>
        /// <param name="template">IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="member">Method name of IHDPTemplate object which implements the query result processing logic.</param>
        /// <param name="clearParams">Specify if the parameters collection should be cleared after the query is executed.</param>
        /// <returns>String array object which contains the representation of the query result.</returns>
        public string[] ExecuteArray(IHDPTemplate template, string member, bool clearParams)
        {
            XPathNavigator xdNav = ExecuteDocument(clearParams).CreateNavigator();
            template.XPathDataSource = xdNav;
            string[] result;

            try
            {
                Type templateType = template.GetType();
                MethodInfo method = templateType.GetMethod(member);
                result = (string[])method.Invoke(template, null);
            }
            catch (Exception ex)
            {
                lastError = ExceptionHelper.GetExceptionMessage(ex, _connection.ConnectionURL);
                ExceptionHelper.DebugException(lastError, ex);

                result = null;
            }

            return result;
        }
        #endregion

        #region Private Methods
        private string PrepareURL(string url)
        {
            StringBuilder _urlString = new StringBuilder(url);
            if (_parameters != null)
            {
                foreach (HDPParameter param in _parameters)
                    _urlString.Append(param.Name.Remove(0, 1) + "=" + param.Value + "&");
            }

            return _urlString.ToString();
        }

        private string PrepareParameters()
        {
            StringBuilder _paramsString = new StringBuilder();
            if (_parameters != null)
            {
                for (int index = 0; index < _parameters.Count; index++)
                {
                    HDPParameter param = _parameters[index];
                    if (ignoreParamsName)
                        _paramsString.Append(param.Value);
                    else
                        _paramsString.Append(param.Name.Remove(0, 1) + ((param.Name.Length == 1) ? "" : "=") + param.Value + (index == _parameters.Count - 1 ? "" : "&"));
                }
            }

            return _paramsString.ToString();
        }

        private void PrepareRequest()
        {
            try
            {
                string url;
                if (_commandType == HDPCommandType.Get)
                    url = PrepareURL(_connection.ConnectionURL);
                else
                    url = _connection.ConnectionURL;

                InitializeRequest(_connection, url);
                request.Method = _commandType == HDPCommandType.Post ? "POST" : "GET";

                if (_commandType == HDPCommandType.Post)
                {
                    byte[] buffer = Encoding.UTF8.GetBytes(PrepareParameters());

                    request.ContentLength = buffer.Length;

                    Stream _reqStream = request.GetRequestStream();
                    _reqStream.Write(buffer, 0, buffer.Length);
                    _reqStream.Flush();
                    _reqStream.Close();
                }
            }
            catch (Exception ex)
            {
                throw (ex);
            }
        }

        private void InitializeRequest(IHDPConnection connection, string url)
        {
            request = (HttpWebRequest)WebRequest.Create(url);

            if (request.CookieContainer != null)
                request.CookieContainer.Add(connection.Cookies);
            else
            {
                request.CookieContainer = new CookieContainer();
                request.CookieContainer.Add(connection.Cookies);
            }

            request.AllowAutoRedirect = connection.AutoRedirect;
            if (_connection.AutoRedirect)
                request.MaximumAutomaticRedirections = connection.MaxAutoRedirects;

            request.Accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/vnd.ms-powerpoint, application/vnd.ms-excel, application/msword, application/x-ms-application, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-ms-xbap, application/x-shockwave-flash, application/pdf, */*";
            request.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
            request.ServicePoint.Expect100Continue = false;

            request.KeepAlive = connection.KeepAlive;
            request.UserAgent = connection.UserAgent;
            request.Referer = connection.Referer;

            request.ContentType = connection.ContentType ?? HDPContentType.AP_XWFORM;

            request.Credentials = CredentialCache.DefaultCredentials;
            request.Proxy = connection.Proxy != null ? connection.Proxy.Create() : null;

            if (connection.Headers != null)
            {
                foreach (HDPConnectionHeader header in connection.Headers)
                    request.Headers.Add(header.Name, header.Value);
            }
        }

        private void ClearParameters()
        {
            if (_parameters != null && _parameters.Count > 0)
                _parameters.Clear();
        }

        private string CreateDataSourceId()
        {
            string dataSource = _connection.ConnectionURL + PrepareParameters();

            MD5CryptoServiceProvider md5Provider = new MD5CryptoServiceProvider();
            byte[] bytes = md5Provider.ComputeHash(Encoding.UTF8.GetBytes(dataSource));
            StringBuilder sbHash = new StringBuilder();

            foreach (byte b in bytes)
                sbHash.Append(b.ToString("x2").ToUpper());

            return sbHash.ToString();
        }

        private byte[] ReadFully(Stream stream, int initialLength, bool block)
        {
            if (initialLength < 1)
                initialLength = 32768;

            byte[] buffer = new byte[initialLength];
            int read = 0;

            int chunk;
            while ((chunk = stream.Read(buffer, read, buffer.Length - read)) > 0)
            {
                if (block && read >= initialLength)
                    break;

                read += chunk;
                if (read == buffer.Length)
                {
                    int nextByte = stream.ReadByte();

                    if (nextByte == -1)
                        return buffer;

                    byte[] newBuffer = new byte[buffer.Length * 2];
                    Array.Copy(buffer, newBuffer, buffer.Length);
                    newBuffer[read] = (byte)nextByte;
                    buffer = newBuffer;
                    read++;
                }
            }

            byte[] ret = new byte[read];
            Array.Copy(buffer, ret, read);
            return ret;
        }

        private string UrlToPath(string url)
        {
            StringBuilder sbUrl = new StringBuilder(url);
            sbUrl = sbUrl.Replace("http://", "");
            sbUrl = sbUrl.Replace(":", "");
            sbUrl = sbUrl.Replace(" ", "");
            sbUrl = sbUrl.Replace("*", "_");
            sbUrl = sbUrl.Replace("/", "_");

            return sbUrl.ToString();
        }
        #endregion

        #region IDisposable Members
        /// <summary>
        /// Dispose HDPCommand object.
        /// </summary>
        public void Dispose()
        {
            dispose();
            GC.SuppressFinalize(this);
        }

        private static void dispose()
        {
        }
        #endregion
    }
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)


Written By
Software Developer (Senior)
Cyprus Cyprus
I am a senior software engineer with over 8 years experience. Have worked for different international software companies using different technologies and programming languages like: C/C++, lotus script, lotus API, C#, ASP.NET, WCF, MS-SQL, Oracle, Domino Server, JavaScript.

Comments and Discussions