Click here to Skip to main content
11,705,052 members (52,986 online)
Click here to Skip to main content

Web site crawling using Visual Studio 2008 webtest file

, 11 Jun 2008 LGPL3 21.8K 326 25
Rate this:
Please Sign up or sign in to vote.
How to run webtest programatically

Background

If you want to get another web site data, you can use HttpWebRequest and easily get page data via HttpWebResponse.But the site reuires form-based logon , Or you need some operation to get desired page, you should manage cookies and hidden variables like viewstate.This is pretty complicated and relyed on site behavior.

So, if you can use webtest file as a direction for crawling, you can easily record your operation and can eliminate programming for each site.

Overview

The step is

  1. Load webtest file and parse it.
  2. Send Request to web site, managing cookies.
  3. Receive Response from website and apply extraction rules.
  4. Send Next Request, when parameter name exists in context, replace parameter value with which in context.
  5. Receive Response, and Loop from 4.

Implementation

Load webtest file and parse.

public void Load(string filename)
{
    requests = new List<RequestData>();

    XmlDocument xmlDoc = new XmlDocument();
    xmlDoc.Load(filename);
    XmlNamespaceManager nsmgr = new XmlNamespaceManager(xmlDoc.NameTable);
    nsmgr.AddNamespace("a",
        "<a href="%22%22http://microsoft.com/schemas/VisualStudio/TeamTest/2006%22%22">http://microsoft.com/schemas/VisualStudio/TeamTest/2006</a>");
    foreach (XmlNode node in xmlDoc.SelectNodes("//a:Request", nsmgr))
    {
        RequestData reqData = new RequestData();
        reqData.Method = node.Attributes["Method"].Value;
        reqData.Url = node.Attributes["Url"].Value;
        reqData.Encoding = node.Attributes["Encoding"].Value;
        foreach (XmlNode formPostParameterNode in node.SelectNodes(
            ".//a:FormPostParameter", nsmgr))
        {
            string paramName = formPostParameterNode.Attributes["Name"].Value;
            string paramValue = formPostParameterNode.Attributes["Value"].Value;
            reqData.FormPostParameters.Add(paramName, paramValue);
        }
        foreach (XmlNode queryStringParameterNode in node.SelectNodes(
            ".//a:QueryStringParameter", nsmgr))
        {
            string paramName = queryStringParameterNode.Attributes["Name"].Value;
            string paramValue = queryStringParameterNode.Attributes["Value"].Value;
            reqData.QueryStringParameters.Add(paramName,paramValue);
        }
        foreach (XmlNode extractionRuleNode in node.SelectNodes(
            "./a:ExtractionRules/a:ExtractionRule",nsmgr))
        {
            ExtractionRuleData ruleData = new ExtractionRuleData();
            ruleData.ClassName = extractionRuleNode.Attributes["Classname"].Value;
            ruleData.VariableName =
                extractionRuleNode.Attributes["VariableName"].Value;
            reqData.ExtractionRules.Add(ruleData);
        }
        requests.Add(reqData);
    }
}

Send Request to web site, managing cookies.Receive Response from website and apply extraction rules.

public string Execute()
{
    string result = null;
    try
    {
        foreach (RequestData reqData in requests)
        {
            result = ExecuteRequest(reqData);
        }
    }
    catch (WebException ex)
    {
        Stream dataStream = ex.Response.GetResponseStream();
        StreamReader reader = new StreamReader(dataStream,
            Encoding.GetEncoding(((HttpWebResponse)ex.Response).CharacterSet));
        result = reader.ReadToEnd();

    }
    return result;
}
#region private
private string ExecuteRequest(RequestData reqData)
{
    HttpWebRequest request = (HttpWebRequest)WebRequest.Create(BuildUrl(reqData));
    request.Method = reqData.Method;
    if (container != null)
    {
        request.CookieContainer = container;
    }
    request.AllowAutoRedirect = true;
    request.Accept = "*/*";
    request.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)";
    request.CookieContainer = container;
    //TODO
    //request.Credentials = CredentialCache.DefaultCredentials;
    if ("POST" == reqData.Method)
    {
        byte[] postData =
           Encoding.GetEncoding(reqData.Encoding).GetBytes(BuildFormData(reqData));
        request.ContentType = "application/x-www-form-urlencoded";
        request.ContentLength = postData.Length;
        using (Stream stream = request.GetRequestStream())
        {
            stream.Write(postData, 0, postData.Length);
        }
    }

    HttpWebResponse response = (HttpWebResponse)request.GetResponse();

    Stream dataStream = response.GetResponseStream();
    StreamReader reader = new StreamReader(
        dataStream,Encoding.GetEncoding(response.CharacterSet));
    string responseFromServer = reader.ReadToEnd();
    reader.Close();
    dataStream.Close();
    response.Close();
    if (reqData.ExtractionRules.Count > 0)
    {
        foreach (ExtractionRuleData ruleData in reqData.ExtractionRules)
        {
            if (ruleData.ClassName.StartsWith(
            "Microsoft.VisualStudio.TestTools.WebTesting.Rules.ExtractHiddenFields"))
            {
                ExtractHiddenField(responseFromServer,ruleData.VariableName);
            }
        }
    }
    return responseFromServer;
}
private string BuildUrl(RequestData reqData)
{
    if (reqData.QueryStringParameters.Count == 0)
    {
        return reqData.Url;
    }
    StringBuilder builder = new StringBuilder();
    builder.Append(reqData.Url);
    bool firstParam = true;
    foreach (string key in reqData.QueryStringParameters.Keys)
    {
        if (firstParam)
        {
            firstParam = false;
            builder.Append("?");
        }
        else
        {
            builder.Append("&");
        }
        builder.Append(key);
        builder.Append("=");
        string value =reqData.QueryStringParameters[key];
        if (Context.ContainsKey(value))
        {
            value = Context[value];
        }
        builder.Append(value);
    }
    return builder.ToString();
}
private string BuildFormData(RequestData reqData)
{
    if (reqData.FormPostParameters.Count == 0)
    {
        return string.Empty;
    }
    StringBuilder builder = new StringBuilder();
    bool firstParam = true;
    foreach (string key in reqData.FormPostParameters.Keys)
    {
        if (firstParam)
        {
            firstParam = false;
        }
        else
        {
            builder.Append("&");
        }

        builder.Append(HttpUtility.UrlEncode(key));
        builder.Append("=");
        string value = reqData.FormPostParameters[key];
        if (Context.ContainsKey(value))
        {
            value = Context[value];
        }
        builder.Append(HttpUtility.UrlEncode(value));
    }
    return builder.ToString();
}
private void ExtractHiddenField(string source, string variableName)
{
    string prefix = "{{$HIDDEN" + variableName + ".";
    string suffix = "}}";
    ParseHTML parse = new ParseHTML();
    parse.Source = source;
    while (!parse.Eof())
    {
        char ch = parse.Parse();
        if (ch == 0)
        {
            AttributeList tag = parse.GetTag();
            if (tag.Name.ToUpper() == "INPUT")
            {
                if (tag["type"] != null && tag["type"].Value.ToUpper() == "HIDDEN")
                {
                    string name = tag["name"].Value;
                    string value = tag["value"].Value;
                    Context.Add(prefix + name + suffix, value);
                }
            }
        }
    }
}
#endregion

Reference

Use below as a HTML Parser.
http://www.developer.com/net/csharp/article.php/2230091

License

This article, along with any associated source code and files, is licensed under The GNU Lesser General Public License (LGPLv3)

Share

About the Author

Masayuki Tanaka
Software Developer
Japan Japan
I am Software Developer currently working at global IT consulting firm in Tokyo, Japan.

I have 9 years of experience in software development especially in Web Application.

I want to improve my English writing and speaking skill.
So,I am looking for English - Japanese language exchange partner in Tokyo area.
If you have some interest, Please email me<m-tanaka@pp.iij4u.or.jp>.

You may also be interested in...

Comments and Discussions

 
Question[My vote of 1] Class Library? What is filename parameter Pin
logistum18-Mar-13 10:35
memberlogistum18-Mar-13 10:35 
Questionproblem in execution Pin
workSmart10-May-12 9:49
memberworkSmart10-May-12 9:49 
GeneralReally superb............... Pin
Shrijeshpk21-Aug-08 22:50
memberShrijeshpk21-Aug-08 22:50 

General General    News News    Suggestion Suggestion    Question Question    Bug Bug    Answer Answer    Joke Joke    Rant Rant    Admin Admin   

Use Ctrl+Left/Right to switch messages, Ctrl+Up/Down to switch threads, Ctrl+Shift+Left/Right to switch pages.

| Advertise | Privacy | Terms of Use | Mobile
Web04 | 2.8.150819.1 | Last Updated 11 Jun 2008
Article Copyright 2008 by Masayuki Tanaka
Everything else Copyright © CodeProject, 1999-2015
Layout: fixed | fluid