Click here to Skip to main content
15,896,606 members
Articles / Code generation

Semi generated crawler

Rate me:
Please Sign up or sign in to vote.
5.00/5 (3 votes)
22 Jun 2012CPOL4 min read 21.6K   599   10  
Leverage Visual studio Web Test Framework for your crawling needs...
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Globalization;
using System.IO;
using System.Text;
using System.Xml;
namespace LightWebTestFramework
{
	internal class DomHelpers
	{
		public static XmlDocument DomFromFile(string fileName)
		{
			XmlTextReader xmlTextReader = new XmlTextReader(fileName);
			XmlDocument xmlDocument = new XmlDocument();
			xmlDocument.Load(xmlTextReader);
			xmlTextReader.Close();
			return xmlDocument;
		}
		public static void DomToFile(XmlDocument doc, string fileName)
		{
			XmlTextWriter xmlTextWriter = new XmlTextWriter(fileName, new UTF8Encoding(true));
			xmlTextWriter.Formatting = Formatting.Indented;
			doc.Save(xmlTextWriter);
			xmlTextWriter.Close();
		}
		public static XmlDocument DomFromStream(Stream stream)
		{
			XmlTextReader xmlTextReader = new XmlTextReader(stream);
			XmlDocument xmlDocument = new XmlDocument();
			xmlDocument.Load(xmlTextReader);
			xmlTextReader.Close();
			return xmlDocument;
		}
		public static void DomToStream(XmlDocument doc, Stream stream)
		{
			XmlDeclaration xmlDeclaration = doc.CreateXmlDeclaration("1.0", null, null);
			xmlDeclaration.Encoding = "UTF-8";
			xmlDeclaration.Standalone = "yes";
			XmlElement documentElement = doc.DocumentElement;
			doc.InsertBefore(xmlDeclaration, documentElement);
			doc.Save(stream);
		}
		public static XmlElement GetRootElement(XmlDocument doc)
		{
			foreach (XmlNode xmlNode in doc.ChildNodes)
			{
				if (xmlNode.NodeType == XmlNodeType.Element)
				{
					return (XmlElement)xmlNode;
				}
			}
			return null;
		}
		public static XmlElement GetFirstChildElement(XmlNode node, string elementName)
		{
			foreach (XmlNode xmlNode in node.ChildNodes)
			{
				if (xmlNode.NodeType == XmlNodeType.Element && xmlNode.Name == elementName)
				{
					return (XmlElement)xmlNode;
				}
			}
			return null;
		}
		public static ArrayList GetChildElements(XmlNode node, string[] elementNames)
		{
			StringCollection stringCollection = new StringCollection();
			for (int i = 0; i < elementNames.Length; i++)
			{
				string value = elementNames[i];
				stringCollection.Add(value);
			}
			ArrayList arrayList = new ArrayList();
			foreach (XmlNode xmlNode in node.ChildNodes)
			{
				if (xmlNode.NodeType == XmlNodeType.Element && stringCollection.Contains(xmlNode.Name))
				{
					arrayList.Add(xmlNode);
				}
			}
			return arrayList;
		}
		public static ArrayList GetChildElements(XmlNode node, string elementName)
		{
			return DomHelpers.GetChildElements(node, new string[]
			{
				elementName
			});
		}
		public static string GetStringAttribute(XmlElement elem, string attributeName)
		{
			return elem.GetAttribute(attributeName);
		}
		public static string[] StringToStringArray(string s)
		{
			string[] array = s.Split(new char[]
			{
				','
			});
			for (int i = 0; i < array.Length; i++)
			{
				int num = 0;
				int j = array[i].IndexOf('%', num);
				if (j > -1)
				{
					StringBuilder stringBuilder = new StringBuilder();
					while (j > -1)
					{
						if (j > 0 && j > num)
						{
							stringBuilder.Append(array[i].Substring(num, j - num));
						}
						if (j < array[i].Length - 1)
						{
							if (array[i][j + 1] == '^')
							{
								stringBuilder.Append(',');
							}
							else
							{
								stringBuilder.Append(array[i][j + 1]);
							}
						}
						num = j + 2;
						j = array[i].IndexOf('%', num);
					}
					if (num < array[i].Length)
					{
						stringBuilder.Append(array[i].Substring(num));
					}
					array[i] = stringBuilder.ToString();
				}
			}
			return array;
		}
		public static string StringArrayToString(string[] strings)
		{
			StringBuilder stringBuilder = new StringBuilder();
			char[] anyOf = new char[]
			{
				',',
				'%'
			};
			for (int i = 0; i < strings.Length; i++)
			{
				string text = strings[i];
				int num = 0;
				for (int j = text.IndexOfAny(anyOf); j > -1; j = text.IndexOfAny(anyOf, num))
				{
					if (j > 0 && j > num)
					{
						stringBuilder.Append(text.Substring(num, j - num));
					}
					stringBuilder.Append('%');
					if (text[j] == ',')
					{
						stringBuilder.Append('^');
					}
					else
					{
						stringBuilder.Append(text[j]);
					}
					num = j + 1;
				}
				if (num < text.Length)
				{
					stringBuilder.Append(text.Substring(num));
				}
				if (i < strings.Length - 1)
				{
					stringBuilder.Append(',');
				}
			}
			return stringBuilder.ToString();
		}
		public static string[] GetStringArrayAttribute(XmlElement elem, string attributeName)
		{
			if (elem == null)
			{
				throw new ArgumentNullException("elem");
			}
			if (attributeName == null)
			{
				throw new ArgumentNullException("attributeName");
			}
			if (elem.GetAttributeNode(attributeName) == null)
			{
				return null;
			}
			string attribute = elem.GetAttribute(attributeName);
			return DomHelpers.StringToStringArray(attribute);
		}
		public static void SetStringArrayAttribute(XmlElement elem, string attributeName, string[] strings)
		{
			if (elem == null)
			{
				throw new ArgumentNullException("elem");
			}
			if (attributeName == null)
			{
				throw new ArgumentNullException("attributeName");
			}
			if (strings == null)
			{
				throw new ArgumentNullException("strings");
			}
			string value = DomHelpers.StringArrayToString(strings);
			elem.SetAttribute(attributeName, value);
		}
		public static List<int> GetIntListAttribute(XmlElement elem, string attributeName)
		{
			if (elem == null)
			{
				throw new ArgumentNullException("elem");
			}
			if (attributeName == null)
			{
				throw new ArgumentNullException("attributeName");
			}
			if (elem.GetAttributeNode(attributeName) == null)
			{
				return null;
			}
			string attribute = elem.GetAttribute(attributeName);
			string[] array = DomHelpers.StringToStringArray(attribute);
			List<int> list = new List<int>(array.Length);
			string[] array2 = array;
			for (int i = 0; i < array2.Length; i++)
			{
				string text = array2[i];
				if (!string.IsNullOrEmpty(text))
				{
					list.Add(int.Parse(text, NumberFormatInfo.InvariantInfo));
				}
			}
			return list;
		}
		public static void SetIntListAttribute(XmlElement elem, string attributeName, List<int> ints)
		{
			if (elem == null)
			{
				throw new ArgumentNullException("elem");
			}
			if (attributeName == null)
			{
				throw new ArgumentNullException("attributeName");
			}
			if (ints == null)
			{
				throw new ArgumentNullException("ints");
			}
			string[] array = new string[ints.Count];
			for (int i = 0; i < ints.Count; i++)
			{
				array[i] = ints[i].ToString(CultureInfo.InvariantCulture);
			}
			string value = DomHelpers.StringArrayToString(array);
			elem.SetAttribute(attributeName, value);
		}
		public static string GetStringAttribute(XmlElement elem, string attributeName, string defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			return DomHelpers.GetStringAttribute(elem, attributeName);
		}
		private static int GetEnumIntAttribute(XmlElement elem, string attributeName, Type enumType)
		{
			if (Enum.GetUnderlyingType(enumType) != typeof(int))
			{
				throw new ArgumentException("Enumeration " + enumType.Name + " must be of type integer.");
			}
			string attribute = elem.GetAttribute(attributeName);
			if (attribute.Length > 0)
			{
				return (int)Enum.Parse(enumType, attribute);
			}
			int[] array = (int[])Enum.GetValues(enumType);
			if (array.Length < 1)
			{
				throw new ArgumentException("Enumeration " + enumType.Name + " has no values.");
			}
			return array[0];
		}
		public static int GetEnumIntAttribute(XmlElement elem, string attributeName, Type enumType, int defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			int result;
			try
			{
				result = DomHelpers.GetEnumIntAttribute(elem, attributeName, enumType);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static decimal GetDecimalAttribute(XmlElement elem, string attributeName, decimal defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			decimal result;
			try
			{
				result = decimal.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static double GetDoubleAttribute(XmlElement elem, string attributeName, double defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			double result;
			try
			{
				result = double.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static float GetFloatAttribute(XmlElement elem, string attributeName, float defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			float result;
			try
			{
				result = float.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static bool GetBoolAttribute(XmlElement elem, string attributeName, bool defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			return elem.GetAttribute(attributeName).ToUpperInvariant() == "TRUE";
		}
		public static int GetIntAttribute(XmlElement elem, string attributeName, int defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			int result;
			try
			{
				result = int.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static uint GetUIntAttribute(XmlElement elem, string attributeName, uint defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			uint result;
			try
			{
				result = uint.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
		public static long GetLongAttribute(XmlElement elem, string attributeName, long defaultValue)
		{
			if (!elem.HasAttribute(attributeName))
			{
				return defaultValue;
			}
			long result;
			try
			{
				result = long.Parse(elem.GetAttribute(attributeName), NumberFormatInfo.InvariantInfo);
			}
			catch
			{
				result = defaultValue;
			}
			return result;
		}
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)


Written By
Software Developer Freelance
France France
I am currently the CTO of Metaco, we are leveraging the Bitcoin Blockchain for delivering financial services.

I also developed a tool to make IaaS on Azure more easy to use IaaS Management Studio.

If you want to contact me, go this way Smile | :)

Comments and Discussions