Web site crawling using Visual Studio 2008 webtest file

• Download DeepWebScraper - 60.3 KB

Background

If you want to get another web site data, you can use HttpWebRequest and easily get page data via HttpWebResponse.But the site reuires form-based logon , Or you need some operation to get desired page, you should manage cookies and hidden variables like viewstate.This is pretty complicated and relyed on site behavior.

So, if you can use webtest file as a direction for crawling, you can easily record your operation and can eliminate programming for each site.

Overview

The step is

1. Load webtest file and parse it.
2. Send Request to web site, managing cookies.
3. Receive Response from website and apply extraction rules.
4. Send Next Request, when parameter name exists in context, replace parameter value with which in context.
5. Receive Response, and Loop from 4.

Implementation

Load webtest file and parse.

        public void Load(string filename)
        {
            requests = new List<RequestData>();
            
            XmlDocument xmlDoc = new XmlDocument();
            xmlDoc.Load(filename);
            XmlNamespaceManager nsmgr = new XmlNamespaceManager(xmlDoc.NameTable);
            nsmgr.AddNamespace("a",
                "%22%22http://microsoft.com/schemas/VisualStudio/TeamTest/2006%22%22">http://microsoft.com/schemas/VisualStudio/TeamTest/2006");
            foreach (XmlNode node in xmlDoc.SelectNodes("//a:Request", nsmgr))
            {
                RequestData reqData = new RequestData();
                reqData.Method = node.Attributes["Method"].Value;
                reqData.Url = node.Attributes["Url"].Value;
                reqData.Encoding = node.Attributes["Encoding"].Value;
                foreach (XmlNode formPostParameterNode in node.SelectNodes(
                    ".//a:FormPostParameter", nsmgr))
                {
                    string paramName = formPostParameterNode.Attributes["Name"].Value;
                    string paramValue = formPostParameterNode.Attributes["Value"].Value;
                    reqData.FormPostParameters.Add(paramName, paramValue);
                }
                foreach (XmlNode queryStringParameterNode in node.SelectNodes( 
                    ".//a:QueryStringParameter", nsmgr))
                {
                    string paramName = queryStringParameterNode.Attributes["Name"].Value;
                    string paramValue = queryStringParameterNode.Attributes["Value"].Value;
                    reqData.QueryStringParameters.Add(paramName,paramValue);
                }
                foreach (XmlNode extractionRuleNode in node.SelectNodes(
                    "./a:ExtractionRules/a:ExtractionRule",nsmgr))
                {
                    ExtractionRuleData ruleData = new ExtractionRuleData();
                    ruleData.ClassName = extractionRuleNode.Attributes["Classname"].Value;
                    ruleData.VariableName =
                        extractionRuleNode.Attributes["VariableName"].Value;
                    reqData.ExtractionRules.Add(ruleData);
                }
                requests.Add(reqData);
            }            
        }

Send Request to web site, managing cookies.Receive Response from website and apply extraction rules.

        public string Execute()
        {
            string result = null;
            try
            {
                foreach (RequestData reqData in requests)
                {
                    result = ExecuteRequest(reqData);
                }
            }
            catch (WebException ex)
            {
                Stream dataStream = ex.Response.GetResponseStream();
                StreamReader reader = new StreamReader(dataStream,
                    Encoding.GetEncoding(((HttpWebResponse)ex.Response).CharacterSet));
                result = reader.ReadToEnd();
                 
            }
            return result;
        }
        #region private
        private string ExecuteRequest(RequestData reqData)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BuildUrl(reqData));
            request.Method = reqData.Method;
            if (container != null)
            {
                request.CookieContainer = container;
            }
            request.AllowAutoRedirect = true;
            request.Accept = "*/*";
            request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
            request.CookieContainer = container;
            //TODO
            //request.Credentials = CredentialCache.DefaultCredentials;
            if ("POST" == reqData.Method)
            {
                byte[] postData =
                   Encoding.GetEncoding(reqData.Encoding).GetBytes(BuildFormData(reqData));
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = postData.Length;
                using (Stream stream = request.GetRequestStream())
                {
                    stream.Write(postData, 0, postData.Length);
                }
            }
            
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            
            Stream dataStream = response.GetResponseStream();
            StreamReader reader = new StreamReader(
                dataStream,Encoding.GetEncoding(response.CharacterSet));
            string responseFromServer = reader.ReadToEnd();
            reader.Close();
            dataStream.Close();
            response.Close();
            if (reqData.ExtractionRules.Count > 0)
            {
                foreach (ExtractionRuleData ruleData in reqData.ExtractionRules)
                {
                    if (ruleData.ClassName.StartsWith(
                    "Microsoft.VisualStudio.TestTools.WebTesting.Rules.ExtractHiddenFields"))
                    {
                        ExtractHiddenField(responseFromServer,ruleData.VariableName);
                    }
                }
            }
            return responseFromServer;
        }
        private string BuildUrl(RequestData reqData)
        {
            if (reqData.QueryStringParameters.Count == 0)
            {
                return reqData.Url;
            }
            StringBuilder builder = new StringBuilder();
            builder.Append(reqData.Url);
            bool firstParam = true;
            foreach (string key in reqData.QueryStringParameters.Keys)
            {
                if (firstParam)
                {
                    firstParam = false;
                    builder.Append("?");
                }
                else
                {
                    builder.Append("&");
                }
                builder.Append(key);
                builder.Append("=");
                string value =reqData.QueryStringParameters[key];
                if (Context.ContainsKey(value))
                {
                    value = Context[value];
                }
                builder.Append(value);
            }
            return builder.ToString();
        }
        private string BuildFormData(RequestData reqData)
        {
            if (reqData.FormPostParameters.Count == 0)
            {
                return string.Empty;
            }
            StringBuilder builder = new StringBuilder();
            bool firstParam = true;
            foreach (string key in reqData.FormPostParameters.Keys)
            {
                if (firstParam)
                {
                    firstParam = false;
                }
                else
                {
                    builder.Append("&");
                }
               
                builder.Append(HttpUtility.UrlEncode(key));
                builder.Append("=");
                string value = reqData.FormPostParameters[key];
                if (Context.ContainsKey(value))
                {
                    value = Context[value];
                }
                builder.Append(HttpUtility.UrlEncode(value));
            }
            return builder.ToString();
        }
        private void ExtractHiddenField(string source, string variableName)
        {
            string prefix = "{{$HIDDEN" + variableName + ".";
            string suffix = "}}";
            ParseHTML parse = new ParseHTML();
            parse.Source = source;
            while (!parse.Eof())
            {
                char ch = parse.Parse();
                if (ch == 0)
                {
                    AttributeList tag = parse.GetTag();
                    if (tag.Name.ToUpper() == "INPUT")
                    {
                        if (tag["type"] != null && tag["type"].Value.ToUpper() == "HIDDEN")
                        {
                            string name = tag["name"].Value;
                            string value = tag["value"].Value;
                            Context.Add(prefix + name + suffix, value);
                        }
                    }
                }
            }
        }
        #endregion

Reference

Use below as a HTML Parser.
http://www.developer.com/net/csharp/article.php/2230091