Click here to Skip to main content
Rate this: bad
good
Please Sign up or sign in to vote.
See more: C# Application Console
I'm writing a C# console application that scrapes data from web pages. This application will go to about 8000 web pages and wait till web page is completly rendered with data.
 
I have it working right now with no async methods and no multithreading.
 
However, I need it to be faster. It only uses about 3%-6% of the CPU, I think because it spends the time.
 

Need to implement wait kind of thing, for ex i have 8000 web url need to render in IE, so that will hit the url in a batch of 100 in parallel and with for completion and wait till its finish then after another 100 and so on.
 

Can anyone point me in the right direction on how to make that line async in .net 4.5 c# and then have my merge method run on complete?
 

--------------------------------------------------------------------------------
using System;
using System.Data;
using Microsoft.SqlServer.Dts.Runtime;
using System.Windows.Forms;
using System.IO;
using System.Data.OleDb;
using System.Data;
using System.Data.SqlClient;
using System.Net;
namespace ST_18c7c9d1477c4c8e8ff78b9832240e61.csproj
{
    [System.AddIn.AddIn("ScriptMain", Version = "1.0", Publisher = "", Description = "")]
    public partial class ScriptMain : Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase
    {
            */
        public void Main()
        {
            Console.WriteLine("hello");
           // string connectionString = "Server=Mayank-Pc;Database=reportserver;User=sa;Password=mayank;Trusted_Connection=False;";
            string connectionString = "Server=Mayank-Pc;Database=reportserver;Trusted_Connection=True;";
            SqlConnection conn = new SqlConnection(connectionString);
            conn.Open();
            Console.WriteLine("done");
            string strSQLQuery = string.Empty;
            strSQLQuery = @"SELECT 'mayank-pc' AS ServerName,C.Path AS ReportPath,'salesorderid=43659'  Parameter,0 MaxTimeDataRetrievalSeconds,0 MinTimeDataRetrievalSeconds,99 TotalCount FROM Catalog C where  path like '/Reports/Sales%'";
            SqlCommand cmd = new SqlCommand(strSQLQuery, conn);
            SqlDataAdapter adapt = new SqlDataAdapter(cmd);
            System.Data.DataTable table = new System.Data.DataTable("allPrograms");
            adapt.Fill(table);
            foreach (DataRow dr in table.Rows)
            {
                //http:// mayank-pc/ReportServer/Pages/ReportViewer.aspx?/Reports/Sales&rs:Command=Render&salesorderid=43659
                string strPath = "http://" + dr["ServerName"].ToString() + "/ReportServer/Pages/ReportViewer.aspx?" + dr["ReportPath"].ToString() + "&rs:Command=Render&" + dr["Parameter"].ToString();
                Console.Write(strPath + "\n");
                //System.Diagnostics.Process.Start("iexplore", strPath);
                WebRequest myRequest = WebRequest.Create(strPath);
                myRequest.Credentials = new NetworkCredential("mayank", "India@1985");
                myRequest.Method = "GET";
                myRequest.PreAuthenticate = true;
                // Return the response. 
                try
                {
                    WebResponse myResponse = myRequest.GetResponse();
Process.Start("iexplore", strpath);
                    Console.Write("Success" + "\n");
                }
                catch (WebException e)
                {
                    Console.Write("Error:" + e.ToString() + "\n");
                }
            }
            Console.Read();
     
                   }
        public SqlConnection objConn { get; set; }
    }
}
 

Posted 13-Mar-13 8:49am
Comments
Sergey Alexandrovich Kryukov at 13-Mar-13 14:02pm
   
Multithreading is not for doing things faster, it is for doing things in parallel. Lack of parallel processing can simply block the work, prevent making decisions on the fly, etc. You need to realize what you really want to do. And explain it.
—SA

1 solution

Rate this: bad
good
Please Sign up or sign in to vote.

Solution 1

Hi
you asked the same question last time.
 
Need to implement Threading in C# console application[^]
 
What didn't work?
 
Jegan
  Permalink  

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

  Print Answers RSS
0 OriginalGriff 587
1 Sergey Alexandrovich Kryukov 544
2 Maciej Los 325
3 DamithSL 223
4 Mathew Soji 195
0 OriginalGriff 7,356
1 Sergey Alexandrovich Kryukov 6,712
2 DamithSL 5,461
3 Manas Bhardwaj 4,916
4 Maciej Los 4,475


Advertise | Privacy | Mobile
Web04 | 2.8.1411023.1 | Last Updated 13 Mar 2013
Copyright © CodeProject, 1999-2014
All Rights Reserved. Terms of Service
Layout: fixed | fluid

CodeProject, 503-250 Ferrand Drive Toronto Ontario, M3C 3G8 Canada +1 416-849-8900 x 100