Click here to Skip to main content
15,888,113 members
Articles / Code generation

Semi generated crawler

Rate me:
Please Sign up or sign in to vote.
5.00/5 (3 votes)
22 Jun 2012CPOL4 min read 21.6K   599   10  
Leverage Visual studio Web Test Framework for your crawling needs...
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Threading;
namespace LightWebTestFramework
{
	[Serializable]
	internal class ByteArrayCache
	{
		private static MD5 s_md5 = new MD5CryptoServiceProvider();
		private Dictionary<Guid, int> m_mapGuidToHandle;
		private Dictionary<int, byte[]> m_mapHandleToBytes;
		private int m_nextHandle;
		internal Dictionary<int, byte[]> HandleToBytesMap
		{
			get
			{
				return this.m_mapHandleToBytes;
			}
			set
			{
				this.m_mapHandleToBytes = value;
			}
		}
		internal int NextHandle
		{
			get
			{
				return this.m_nextHandle;
			}
			set
			{
				this.m_nextHandle = value;
			}
		}
		public ByteArrayCache()
		{
			this.m_mapGuidToHandle = new Dictionary<Guid, int>();
			this.m_mapHandleToBytes = new Dictionary<int, byte[]>();
			this.m_nextHandle = 0;
		}
		public byte[] GetBytesForHandle(int handle)
		{
			if (this.m_mapHandleToBytes.ContainsKey(handle))
			{
				return this.DecompressBytes(this.m_mapHandleToBytes[handle]);
			}
			return new byte[0];
		}
		public int GetHandleForBytes(byte[] bytes)
		{
			Guid key = ByteArrayCache.ComputeHash(bytes);
			if (!this.m_mapGuidToHandle.ContainsKey(key))
			{
				int num = this.m_nextHandle++;
				this.m_mapGuidToHandle.Add(key, num);
				this.m_mapHandleToBytes.Add(num, this.CompressBytes(bytes));
				return num;
			}
			return this.m_mapGuidToHandle[key];
		}
		private static Guid ComputeHash(byte[] bytes)
		{
			MD5 obj;
			Monitor.Enter(obj = ByteArrayCache.s_md5);
			byte[] b;
			try
			{
				b = ByteArrayCache.s_md5.ComputeHash(bytes);
			}
			finally
			{
				Monitor.Exit(obj);
			}
			return new Guid(b);
		}
		private byte[] CompressBytes(byte[] bytesUncompressed)
		{
			MemoryStream memoryStream = new MemoryStream();
			GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Compress);
			gZipStream.Write(bytesUncompressed, 0, bytesUncompressed.Length);
			gZipStream.Close();
			byte[] result = memoryStream.ToArray();
			memoryStream.Close();
			return result;
		}
		private byte[] DecompressBytes(byte[] bytesCompressed)
		{
			MemoryStream stream = new MemoryStream(bytesCompressed);
			GZipStream gZipStream = new GZipStream(stream, CompressionMode.Decompress);
			MemoryStream memoryStream = new MemoryStream();
			byte[] array = new byte[2048];
			int num;
			do
			{
				num = gZipStream.Read(array, 0, array.Length);
				if (num > 0)
				{
					memoryStream.Write(array, 0, num);
				}
			}
			while (num == array.Length);
			gZipStream.Close();
			byte[] result = memoryStream.ToArray();
			memoryStream.Close();
			return result;
		}
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)


Written By
Software Developer Freelance
France France
I am currently the CTO of Metaco, we are leveraging the Bitcoin Blockchain for delivering financial services.

I also developed a tool to make IaaS on Azure more easy to use IaaS Management Studio.

If you want to contact me, go this way Smile | :)

Comments and Discussions