Click here to Skip to main content
15,881,413 members
Articles / Programming Languages / Visual Basic

C#/VB - Automated WebSpider / WebRobot

Rate me:
Please Sign up or sign in to vote.
4.66/5 (35 votes)
15 Mar 20045 min read 144.2K   2.8K   170  
Build a flexible WebRobot and process an entire site using a WebSpider
<VisualStudioProject>
    <VisualBasic
        ProjectType = "Local"
        ProductVersion = "7.10.3077"
        SchemaVersion = "2.0"
        ProjectGuid = "{F7E1F3D1-A99D-45F5-A330-49A1BCF8F941}"
    >
        <Build>
            <Settings
                ApplicationIcon = ""
                AssemblyKeyContainerName = ""
                AssemblyName = "MfService_VB"
                AssemblyOriginatorKeyFile = ""
                AssemblyOriginatorKeyMode = "None"
                DefaultClientScript = "JScript"
                DefaultHTMLPageLayout = "Grid"
                DefaultTargetSchema = "IE50"
                DelaySign = "false"
                OutputType = "Library"
                OptionCompare = "Binary"
                OptionExplicit = "On"
                OptionStrict = "Off"
                RootNamespace = "MfService_VB"
                StartupObject = ""
            >
                <Config
                    Name = "Debug"
                    BaseAddress = "285212672"
                    ConfigurationOverrideFile = ""
                    DefineConstants = ""
                    DefineDebug = "true"
                    DefineTrace = "true"
                    DebugSymbols = "true"
                    IncrementalBuild = "true"
                    Optimize = "false"
                    OutputPath = "bin\"
                    RegisterForComInterop = "false"
                    RemoveIntegerChecks = "false"
                    TreatWarningsAsErrors = "false"
                    WarningLevel = "1"
                />
                <Config
                    Name = "Release"
                    BaseAddress = "285212672"
                    ConfigurationOverrideFile = ""
                    DefineConstants = ""
                    DefineDebug = "false"
                    DefineTrace = "true"
                    DebugSymbols = "false"
                    IncrementalBuild = "false"
                    Optimize = "true"
                    OutputPath = "bin\"
                    RegisterForComInterop = "false"
                    RemoveIntegerChecks = "false"
                    TreatWarningsAsErrors = "false"
                    WarningLevel = "1"
                />
            </Settings>
            <References>
                <Reference
                    Name = "System"
                    AssemblyName = "System"
                />
                <Reference
                    Name = "System.Data"
                    AssemblyName = "System.Data"
                />
                <Reference
                    Name = "System.XML"
                    AssemblyName = "System.Xml"
                />
                <Reference
                    Name = "System.Windows.Forms"
                    AssemblyName = "System.Windows.Forms"
                    HintPath = "..\..\..\..\WINDOWS\Microsoft.NET\Framework\v1.1.4322\System.Windows.Forms.dll"
                />
                <Reference
                    Name = "MfUtil"
                    Project = "{687F7EB2-53C2-4B58-9C04-134EBB3FC8CD}"
                    Package = "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"
                />
            </References>
            <Imports>
                <Import Namespace = "Microsoft.VisualBasic" />
                <Import Namespace = "System" />
                <Import Namespace = "System.Collections" />
                <Import Namespace = "System.Data" />
                <Import Namespace = "System.Diagnostics" />
            </Imports>
        </Build>
        <Files>
            <Include>
                <File
                    RelPath = "AssemblyInfo.vb"
                    SubType = "Code"
                    BuildAction = "Compile"
                />
                <File
                    RelPath = "Vb\Service\WebSpider\IWebPageProcessor.vb"
                    SubType = "Code"
                    BuildAction = "Compile"
                />
                <File
                    RelPath = "Vb\Service\WebSpider\WebPageProcessor.vb"
                    SubType = "Code"
                    BuildAction = "Compile"
                />
                <File
                    RelPath = "Vb\Service\WebSpider\WebPageState.vb"
                    SubType = "Code"
                    BuildAction = "Compile"
                />
                <File
                    RelPath = "Vb\Service\WebSpider\WebSpider.vb"
                    SubType = "Code"
                    BuildAction = "Compile"
                />
            </Include>
        </Files>
    </VisualBasic>
</VisualStudioProject>

By viewing downloads associated with this article you agree to the Terms of Service and the article's licence.

If a file you wish to view isn't highlighted, and is a text file (not binary), please let us know and we'll add colourisation support for it.

License

This article has no explicit license attached to it but may contain usage terms in the article text or the download files themselves. If in doubt please contact the author via the discussion board below.

A list of licenses authors might use can be found here


Written By
Web Developer
Australia Australia
I have been programming commercially since 1990, my last two major roles have been Architect/Lead Developer for an online bank and Australia's largest consumer finance comparison portal.

On the side I am a Forex Currency Trader and actively develop tools and applications for Currency Traders.

I have just launched a personal blog at www.davidcruwys.com and a website targeting Foreign Exchange traders at www.my-trading-journal.com

Comments and Discussions