Comments by rameshk14 (Top 14 by date)

hi tomas takac

yes, i did some times its executing, but after some time (10 to 15 minutes)not working... and i am getting null value in nextHtmlNode

this code is executing some some times not(executing else condition).... please help me


Here is my code....


private void searchAndParse(string searchStr)
{
Dictionary<string, string=""> queryPara;
Dictionary<string, string=""> headerDict;
string curGoogleSearchUrl;
string curSearchRespHtml;
string curReferer;
//string ei = "";

//1. for get cookie and related url
string googleUrl = "http://www.google.co.in/";
string tmpRespHtml = crifanLib.getUrlRespHtml(googleUrl);
//processXjs(tmpRespHtml);

//2. do first search
//weight loss site:.edu inurl:blog "post a comment" -"you must be logged in"
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

//http://www.google.com.hk/search
//?newwindow=1
//&safe=strict
//&site=
//&source=hp
//&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22
//&btnK=Google+Search
string firstGoogleSearchUrl = "http://www.google.co.in/search?";
queryPara = new Dictionary<string, string="">();
queryPara.Add("newwindow", "1");
queryPara.Add("safe", "strict");
queryPara.Add("source", "hp");
queryPara.Add("q", searchStr);
queryPara.Add("btnK", "Google Search");

queryPara.Add("site", "");
//queryPara.Add("site", "webhp");

//http://www.google.com.hk/search?newwindow=1&safe=strict&hl=zh-CN&site=webhp&source=hp&q=unit&btnK=Google+%E6%90%9C%E7%B4%A2


//firstGoogleSearchUrl += crifanLib.quoteParas(queryPara);
firstGoogleSearchUrl += crifanLib.quoteParas(queryPara, false);
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3a.edu+inurl%3ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

curReferer = googleUrl;
curGoogleSearchUrl = firstGoogleSearchUrl;

//3. continue to search current page and next page ...
while(needContinueSearch)
{
headerDict = new Dictionary<string,string>();
headerDict.Add("referer", curReferer);
curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl, headerDict: headerDict);
//curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl);

//processXjs(tmpRespHtml);

parseSinglePageHtml(curSearchRespHtml);
if (!needContinueSearch)
{
break;
}

HtmlAgilityPack.HtmlDocument htmlDoc = crifanLib.htmlToHtmlDoc(curSearchRespHtml);
HtmlNode rootHtmlNode = htmlDoc.DocumentNode;
HtmlNode nextHtmlNode = rootHtmlNode.SelectSingleNode("//a[@id='pnnext' and @class='pn']");
if(nextHtmlNode != null)
{
//before update google url, store it to refer for next search use
curReferer = curGoogleSearchUrl;

//Method 2: extract net page url
string hrefStr = nextHtmlNode.Attributes["href"].Value;
//string decodedUrl = HttpUtility.HtmlDecode(hrefStr);
//string decodedUrl = HttpUtility.UrlDecode(hrefStr);

//string nextPageUrl = googleUrl + hrefStr;
string encodedUrl = "http://www.google.co.in" + hrefStr;

string htmlDecoded = HttpUtility.HtmlDecode(encodedUrl);

curGoogleSearchUrl = htmlDecoded;

Thank you for your reply

Here is code


private void btnSearchFootprint_Click(object sender, EventArgs e)
{
if (curSearchStatus == search_status.SEARCH_STATUS_STOPPED)
{
crifanLib.dgvClearContent(dataGridView1);

needContinueSearch = true;


string searchStr = "";
searchStr += textBox1.Text;
searchStr += getCurSearchTail();

webBrowser1.Navigate("http://www.google.com/search?q=" + searchStr);
//initSearchResultGridView();
searchAndParse(searchStr);
}

}


private void searchAndParse(string searchStr)
{
Dictionary<string,> queryPara;
Dictionary<string,> headerDict;
string curGoogleSearchUrl;
string curSearchRespHtml;
string curReferer;
//string ei = "";

//1. for get cookie and related url
string googleUrl = "http://www.google.com.hk/";
string tmpRespHtml = crifanLib.getUrlRespHtml(googleUrl);
//processXjs(tmpRespHtml);

//2. do first search
//weight loss site:.edu inurl:blog "post a comment" -"you must be logged in"
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

//http://www.google.com.hk/search
//?newwindow=1
//&safe=strict
//&site=
//&source=hp
//&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22
//&btnK=Google+Search
string firstGoogleSearchUrl = "http://www.google.com.hk/search?";
queryPara = new Dictionary<string,>();
queryPara.Add("newwindow", "1");
queryPara.Add("safe", "strict");
queryPara.Add("source", "hp");
queryPara.Add("q", searchStr);
queryPara.Add("btnK", "Google Search");

queryPara.Add("site", "");
//queryPara.Add("site", "webhp");

//http://www.google.com.hk/search?newwindow=1&safe=strict&hl=zh-CN&site=webhp&source=hp&q=unit&btnK=Google+%E6%90%9C%E7%B4%A2


//firstGoogleSearchUrl += crifanLib.quoteParas(queryPara);
firstGoogleSearchUrl += crifanLib.quoteParas(queryPara, false);
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3a.edu+inurl%3ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

curReferer = googleUrl;
curGoogleSearchUrl = firstGoogleSearchUrl;

//3. continue to search current page and next page ...
while(needContinueSearch)
{
headerDict = new Dictionary<string,string>();
headerDict.Add("referer", curReferer);
curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl, headerDict: headerDict);
//curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl);

//processXjs(tmpRespHtml);

parseSinglePageHtml(curSearchRespHtml);
if (!needContinueSearch)
{
break;
}

////window.google={kEI:"XZp2UczlE8nYigeK5IHgCg"
////get ei for next page
//if(crifanLib.extractSingleStr(@"window.google={kEI:""(\w+)""", curSearchRespHtml, out ei))
//{

//}

//check need get more page or not
//

Thank you for your reply

Here is code


private void btnSearchFootprint_Click(object sender, EventArgs e)
{
if (curSearchStatus == search_status.SEARCH_STATUS_STOPPED)
{
crifanLib.dgvClearContent(dataGridView1);

needContinueSearch = true;


string searchStr = "";
searchStr += textBox1.Text;
searchStr += getCurSearchTail();

webBrowser1.Navigate("http://www.google.com/search?q=" + searchStr);
//initSearchResultGridView();
searchAndParse(searchStr);
}

}


private void searchAndParse(string searchStr)
{
Dictionary<string, string=""> queryPara;
Dictionary<string, string=""> headerDict;
string curGoogleSearchUrl;
string curSearchRespHtml;
string curReferer;
//string ei = "";

//1. for get cookie and related url
string googleUrl = "http://www.google.com.hk/";
string tmpRespHtml = crifanLib.getUrlRespHtml(googleUrl);
//processXjs(tmpRespHtml);

//2. do first search
//weight loss site:.edu inurl:blog "post a comment" -"you must be logged in"
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

//http://www.google.com.hk/search
//?newwindow=1
//&safe=strict
//&site=
//&source=hp
//&q=weight+loss+site%3A.edu+inurl%3Ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22
//&btnK=Google+Search
string firstGoogleSearchUrl = "http://www.google.com.hk/search?";
queryPara = new Dictionary<string, string="">();
queryPara.Add("newwindow", "1");
queryPara.Add("safe", "strict");
queryPara.Add("source", "hp");
queryPara.Add("q", searchStr);
queryPara.Add("btnK", "Google Search");

queryPara.Add("site", "");
//queryPara.Add("site", "webhp");

//http://www.google.com.hk/search?newwindow=1&safe=strict&hl=zh-CN&site=webhp&source=hp&q=unit&btnK=Google+%E6%90%9C%E7%B4%A2


//firstGoogleSearchUrl += crifanLib.quoteParas(queryPara);
firstGoogleSearchUrl += crifanLib.quoteParas(queryPara, false);
//http://www.google.com.hk/search?newwindow=1&safe=strict&site=&source=hp&q=weight+loss+site%3a.edu+inurl%3ablog+%22post+a+comment%22+-%22you+must+be+logged+in%22&btnK=Google+Search

curReferer = googleUrl;
curGoogleSearchUrl = firstGoogleSearchUrl;

//3. continue to search current page and next page ...
while(needContinueSearch)
{
headerDict = new Dictionary<string,string>();
headerDict.Add("referer", curReferer);
curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl, headerDict: headerDict);
//curSearchRespHtml = crifanLib.getUrlRespHtml(curGoogleSearchUrl);

//processXjs(tmpRespHtml);

parseSinglePageHtml(curSearchRespHtml);
if (!needContinueSearch)
{
break;
}

////window.google={kEI:"XZp2UczlE8nYigeK5IHgCg"
////get ei for next page
//if(crifanLib.extractSingleStr(@"window.google={kEI:""(\w+)""", curSearchRespHtml, out ei))
//{

//}

//check need get more page or not
//

thank you just now the problem get solved...

thank you for your reply, you said that change my code as follows....


string respHtml = "";

HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout, postDataStr);
if (resp != null)
{
var respStream = resp.GetResponseStream();

if (respStream != null)
{
using (var sr = String.IsNullOrEmpty(charset)
? new StreamReader(respStream)
: new StreamReader(respStream, Encoding.GetEncoding(charset)))
{
respHtml = sr.ReadToEnd();
}

}
}

return respHtml;




after changing my code as above i am getting the same error at as Follows.....



private void parseSinglePageHtml(string singlePageGoogleHtml)
{
//

Type-2 Diabetes an Autoimmune Disease? - Hemoblogin - School ...

HtmlAgilityPack.HtmlDocument htmlDoc = crifanLib.htmlToHtmlDoc(singlePageGoogleHtml);
HtmlNode rootHtmlNode = htmlDoc.DocumentNode;
HtmlNodeCollection h3aHtmlNodes = rootHtmlNode.SelectNodes("//h3[@class='r']/a");
foreach (HtmlNode h3aNode in h3aHtmlNodes) //Here is an error
{
if (needContinueSearch)
{
searchItemInfo singleItemInfo = new searchItemInfo();

//InnerHtml
//"Losing weight and belly fat improves sleep - Harvard Health ..."
//InnerText:
//"Losing weight and belly fat improves sleep - Harvard Health ..."
string undecodedTitle = h3aNode.InnerText;
//singleItemInfo.title = HttpUtility.HtmlDecode(undecodedTitle);

singleItemInfo.url = h3aNode.Attributes["href"].Value;

processEachSearchItem(singleItemInfo);
}
else
{
break;
}
}

}


even if i try for any changes in this code i am getting same error some where else

please try to help me....

yeah sure....


here is my code

public string getUrlRespHtml(string url,
Dictionary<string,> headerDict,
string charset,
Dictionary<string,> postDict,
int timeout,
string postDataStr)
{
string respHtml = "";

//HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout);
HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout, postDataStr);

//long realRespLen = resp.ContentLength;

StreamReader sr;
if ((charset != null) && (charset != ""))
{
Encoding htmlEncoding = Encoding.GetEncoding(charset);
sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);

}
else
{
sr = new StreamReader(resp.GetResponseStream()); //Here I am Getting Run time Error Like "object reference not set to an instance of an object. in c#"
}
respHtml = sr.ReadToEnd();

return respHtml;
}

yeah sure....


here is my code

public string getUrlRespHtml(string url,
Dictionary<string,> headerDict,
string charset,
Dictionary<string,> postDict,
int timeout,
string postDataStr)
{
string respHtml = "";

//HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout);
HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout, postDataStr);

//long realRespLen = resp.ContentLength;

StreamReader sr;
if ((charset != null) && (charset != ""))
{
Encoding htmlEncoding = Encoding.GetEncoding(charset);
sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);

}
else
{
sr = new StreamReader(resp.GetResponseStream()); //Here I am Getting Run time Error Like "object reference not set to an instance of an object. in c#"
}
respHtml = sr.ReadToEnd();

return respHtml;
}

thank you for this suggestion...

i tried in that way...

here is my code

public string getUrlRespHtml(string url,
Dictionary<string, string=""> headerDict,
string charset,
Dictionary<string, string=""> postDict,
int timeout,
string postDataStr)
{
string respHtml = "";

//HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout);
HttpWebResponse resp = getUrlResponse(url, headerDict, postDict, timeout, postDataStr);

//long realRespLen = resp.ContentLength;

StreamReader sr;
if ((charset != null) && (charset != ""))
{
Encoding htmlEncoding = Encoding.GetEncoding(charset);
sr = new StreamReader(resp.GetResponseStream(), htmlEncoding);
respHtml = sr.ReadToEnd();
}
else
{
sr = new StreamReader(resp.GetResponseStream()); //Here I am Getting Run time Error Like "object reference not set to an instance of an object. in c#"
}
respHtml = sr.ReadToEnd();

return respHtml;
}