c# .net 4.5 async / multithread?

See more:

C#

application

console

I'm writing a C# console application that scrapes data from web pages. This application will go to about 8000 web pages and wait till web page is completly rendered with data.

I have it working right now with no async methods and no multithreading.

However, I need it to be faster. It only uses about 3%-6% of the CPU, I think because it spends the time.


Need to implement wait kind of thing, for ex i have 8000 web url need to render in IE, so that will hit the url in a batch of 100 in parallel and with for completion and wait till its finish then after another 100 and so on.


Can anyone point me in the right direction on how to make that line async in .net 4.5 c# and then have my merge method run on complete?


--------------------------------------------------------------------------------

using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using System.Data.OleDb;
using System.Data;
using System.Data.SqlClient;
using System.Net;
namespace ST_18c7c9d1477c4c8e8ff78b9832240e61.csproj
{
    [System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
    public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
    {
            */
        public void Main()
        {
            Console.WriteLine("hello");
           // string connectionString = "Server=Mayank-Pc;Database=reportserver;User=sa;Password=mayank;Trusted_Connection=False;";
            string connectionString = "Server=Mayank-Pc;Database=reportserver;Trusted_Connection=True;";
            SqlConnection conn = new SqlConnection(connectionString);
            conn.Open();
            Console.WriteLine("done");
            string strSQLQuery = string.Empty;
            strSQLQuery = @"SELECT 'mayank-pc' AS ServerName,C.Path AS ReportPath,'salesorderid=43659'  Parameter,0 MaxTimeDataRetrievalSeconds,0 MinTimeDataRetrievalSeconds,99 TotalCount FROM Catalog C where  path like '/Reports/Sales%'";
            SqlCommand cmd = new SqlCommand(strSQLQuery, conn);
            SqlDataAdapter adapt = new SqlDataAdapter(cmd);
            System.Data.DataTable table = new System.Data.DataTable("allPrograms");
            adapt.Fill(table);
            foreach (DataRow dr in table.Rows)
            {
                //http:// mayank-pc/ReportServer/Pages/ReportViewer.aspx?/Reports/Sales&rs:Command=Render&salesorderid=43659
                string strPath = "http://" + dr["ServerName"].ToString() + "/ReportServer/Pages/ReportViewer.aspx?" + dr["ReportPath"].ToString() + "&rs:Command=Render&" + dr["Parameter"].ToString();
                Console.Write(strPath + "\n");
                //System.Diagnostics.Process.Start("iexplore", strPath);
                WebRequest myRequest = WebRequest.Create(strPath);
                myRequest.Credentials = new NetworkCredential("mayank", "India@1985");
                myRequest.Method = "GET";
                myRequest.PreAuthenticate = true;
                // Return the response. 
                try
                {
                    WebResponse myResponse = myRequest.GetResponse();
Process.Start("iexplore", strpath);
                    Console.Write("Success" + "\n");
                }
                catch (WebException e)
                {
                    Console.Write("Error:" + e.ToString() + "\n");
                }
            }
            Console.Read();
     
                   }
        public SqlConnection objConn { get; set; }
    }
}