Click here to Skip to main content
15,893,588 members
Articles / Programming Languages / C#

Harvesting Web Content into MHTML Archive

Rate me:
Please Sign up or sign in to vote.
4.69/5 (14 votes)
18 Feb 20068 min read 62K   729   49  
Capture and archive web resources by saving them in RFC-2557 (MHTML) compliant format. This library includes a framework for augmenting the capture process and allowing programmatic control, from downloading through saving. A replacement for CDOSYS/CDONTS.
using System;
using System.Xml;
using System.Collections.Generic;
using System.Text;
using NUnit.Framework;
using System.IO;
using Rilling.MhtmlLib.Media;
using Rilling.MhtmlLib.Media.Artifacts;
using Rilling.MhtmlLib.Media.Resolvers;

namespace Rilling.Web.Mhtml.TestHarness
{
	[TestFixture()]
	public class ImageLinkResolverTest
	{
		string[] testVals1 = new string[]
        { 
			"<img src=\"myimage.jpg\"/>",
			"<img src=\"myimage.jpg\" xxx=\"nothing\"/>",
			"<img xxx=\"nothing\" src=\"myimage.jpg\"/>",
			"<img src='myimage.jpg'/>",
			"<img src=myimage.jpg/>",

			"<input src=\"myimage.jpg\"/>",
			"<input src=\"myimage.jpg\" xxx=\"nothing\"/>",
			"<input xxx=\"nothing\" src=\"myimage.jpg\"/>",
			"<input src='myimage.jpg'/>",
			"<input src=myimage.jpg/>",

			"<input src=\"myimage.jpg\"/>",
			"<input src=\"myimage.jpg\" xxx=\"nothing\"/>",
			"<input xxx=\"nothing\" src=\"myimage.jpg\"/>",
			"<input src='myimage.jpg'/>",
			"<input src=myimage.jpg/>",

			"<input src=\"myimage.jpg\"/>",
			"<input src=\"myimage.jpg\" xxx=\"nothing\"/>",
			"<input xxx=\"nothing\" src=\"myimage.jpg\"/>",
			"<input src='myimage.jpg'/>",
			"<input src=myimage.jpg/>" 
		};

		ImageLinkResolver testObj = null;

		[SetUp]
		public void InitTest()
		{
			testObj = new ImageLinkResolver();
		}

		[Test]
		public void CheckDlt_EvaluateCssFiles()
		{
			Assert.IsTrue(testObj.EvaluateCssFiles);
		}

		[Test]
		public void CheckDlt_EvaluateHtmlFiles()
		{
			Assert.IsTrue(testObj.EvaluateHtmlFiles);
		}

		[Test]
		public void CheckDlt_IncludeBackgroundImages()
		{
			Assert.IsTrue(testObj.IncludeBackgroundImages);
		}

		[Test]
		public void CheckDlt_IncludeForegroundImages()
		{
			Assert.IsTrue(testObj.IncludeForegroundImages);
		}

		[Test]
		public void Set_EvaluateCssFiles()
		{
			bool testVal = !testObj.EvaluateCssFiles;

			testObj.EvaluateCssFiles = testVal;

			Assert.AreEqual(testVal, testObj.EvaluateCssFiles);
		}

		[Test]
		public void Set_EvaluateHtmlFiles()
		{
			bool testVal = !testObj.EvaluateHtmlFiles;

			testObj.EvaluateHtmlFiles = testVal;

			Assert.AreEqual(testVal, testObj.EvaluateHtmlFiles);
		}

		[Test]
		public void Set_IncludeBackgroundImages()
		{
			bool testVal = !testObj.IncludeBackgroundImages;

			testObj.IncludeBackgroundImages = testVal;

			Assert.AreEqual(testVal, testObj.IncludeBackgroundImages);
		}

		[Test]
		public void Set_IncludeForegroundImages()
		{
			bool testVal = !testObj.IncludeForegroundImages;

			testObj.IncludeForegroundImages = testVal;

			Assert.AreEqual(testVal, testObj.IncludeForegroundImages);
		}

		[Test]
		[ExpectedException(typeof(ArgumentNullException))]
		public void Call_LocateReferences_NullParam_ContentData()
		{
			testObj.LocateReferences(null, new Uri("http://www.rilling.net/"));
		}

		[Test]
		[ExpectedException(typeof(ArgumentNullException))]
		public void Call_LocateReferences_NullParam_BaseUrl()
		{
			testObj.LocateReferences(new TextArtifact(new MemoryStream()), null);
		}

		private void CallLocateReferencesHelper(string baseUri,
											   string[] testVals,
											   string expected)
		{
			foreach (string testVal in testVals)
			{
				//
				// Prepare the artifact.
				//
				MemoryStream stream = new MemoryStream();
				StreamWriter writer = new StreamWriter(stream);
				writer.Write(testVal);
				writer.Flush();
				TextArtifact artifact = new TextArtifact(stream);
				artifact.ContentType = ContentTypes.Html;

				//
				// Try to locate and validate the found link.
				//
				LinkIdentification[] results =
					testObj.LocateReferences(artifact, new Uri(baseUri));

				//
				// Make sure it is as expected.
				//
				Assert.AreEqual(1, results.Length, "Unable to locate any links (" + testVal + ")");
				Assert.AreEqual(expected, results[0].Url.ToString(), "Unable to locate any links (" + testVal + ")");
			}
		}

		[Test]
		[ExpectedException(typeof(ArgumentNullException))]
		public void Call_InitializeContext_NullParam()
		{
			testObj.InitializeContext(null);
		}

		[Test]
		public void Call_InitializeContext()
		{
			XmlDocument testVal = new XmlDocument();
			testVal.LoadXml("<context evalCss=\"False\" evalHtml=\"False\" includeBackImg=\"False\" includeForeImg=\"False\"/>");

			testObj.InitializeContext(testVal.FirstChild);

			Assert.AreEqual(false, testObj.EvaluateCssFiles);
			Assert.AreEqual(false, testObj.EvaluateHtmlFiles);
			Assert.AreEqual(false, testObj.IncludeBackgroundImages);
			Assert.AreEqual(false, testObj.IncludeForegroundImages);
		}

		[Test()]
		public void Call_LocateReferences_FolderBaseUri()
		{
			CallLocateReferencesHelper("http://www.rilling.net/nowhere/",
									   testVals1,
									   "http://www.rilling.net/nowhere/myimage.jpg");
		}

		[Test()]
		public void Call_LocateReferences_FileBaseUri()
		{
			CallLocateReferencesHelper("http://www.rilling.net/nowhere",
									   testVals1,
									   "http://www.rilling.net/myimage.jpg");
		}
	}
}

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here


Written By
Software Developer (Senior)
United States United States
This member has not yet provided a Biography. Assume it's interesting and varied, and probably something to do with programming.

Comments and Discussions