How do I download a file from a website using httpwebrequest that has been redirected to a page with an "agree" button

Question

0.00/5 (No votes)

See more:

HttpWebRequest request = (HttpWebRequest)WebRequest.Create(ACHJobsConfig.FederalReserveBankDirectoryURL);

            // Set some reasonable limits on resources used by this request
            request.MaximumAutomaticRedirections = 4;
            request.MaximumResponseHeadersLength = 4;
            // Set credentials to use for this request.
            request.Credentials = CredentialCache.DefaultCredentials;
                                    
            HttpWebResponse response = null;
            try
            {
                // This does the actual downloading of the file
                response = (HttpWebResponse)request.GetResponse();               
            }
            catch (Exception ex)
            {
                Logger.Warn(ex.Message);
                return null;
            }

            if (response == null)
            {
                Logger.Warn("Attempt to download ACH directory file did not return a response.");
                return null;
            }

            // Here we are going to read the contents of the downloaded file
            // one line/record at a time. Each line/record defines a bank routing number.
            // The format of the record is documented in the FedACHDirectoryRecord class.
            StreamReader readStream = null;
            List<FedACHDirectoryRecord> rnList = new List<FedACHDirectoryRecord>();
            try
            {
                // Get the stream associated with the response.
                Stream receiveStream = response.GetResponseStream();

                // Pipes the stream to a higher level stream reader with the required encoding format. 
                readStream = new StreamReader(receiveStream, Encoding.UTF8);

                // Read all the lines out of the downloaded file stream.
                // Each line defines a bank routing number (a banking institution).
                string text = null;
                while ((text = readStream.ReadLine()) != null)
                {
                    // Wrap the line in a record instance (provides access to fields)
                    FedACHDirectoryRecord rec = new FedACHDirectoryRecord(text);
                    // Build the list, record by record
                    rnList.Add(rec);
                }
            }
            catch (Exception ex)
            {
                Logger.Warn(ex.Message);
                return null;
            }
            finally
            {
                // Clean up time
                if (readStream != null)
                {
                    readStream.Close();
                }
                response.Close();
            }

            // Here's what we downloaded
            Logger.InfoFormat("{0} records downloaded from {1}", rnList.Count, ACHJobsConfig.FederalReserveBankDirectoryURL);

Posted 26-Mar-15 7:56am

Member 11027268

Add a Solution

2 solutions

Solution 2

I submitted the above code, just in case anyone else needs help.

Posted 31-Mar-15 7:45am

Member 11027268

Add a Solution

Add your solution here

Treat my content as plain text, not as HTML

Preview 0

…

Existing Members

Sign in to your account

...or Join us

Download, Vote, Comment, Publish.

Your Email
Password
Forgot your password?

Your Email
This email is in use. Do you need your password?
Optional Password

I have read and agree to the Terms of Service and Privacy Policy
Please subscribe me to the CodeProject newsletters

When answering a question please:

Read the question carefully.
Understand that English isn't everyone's first language so be lenient of bad spelling and grammar.
If a question is poorly phrased then either ask for clarification, ignore it, or edit the question and fix the problem. Insults are not welcome.
Don't tell someone to read the manual. Chances are they have and don't get it. Provide an answer or move on to the next question.

Let's work to help developers, not make them feel stupid.

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)

Member 11027268 · Accepted Answer · 2015-03-31T07:42:00

// We will need to use Cookies
            CookieContainer cookies = new CookieContainer();
            List<FedACHDirectoryRecord> rnList = new List<FedACHDirectoryRecord>();
            try
            {
                // Could not use https as there is a certificate issue.
                // Fell back to http.
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(ACHJobsConfig.FederalReserveBankDirectoryURL);
                // cookies for session (will be null at this point)
                request.CookieContainer = cookies;
                // Set some reasonable limits on resources used by this request
                request.MaximumAutomaticRedirections = 4;
                request.MaximumResponseHeadersLength = 4;
                // Set credentials to use for this request.
                request.Credentials = CredentialCache.DefaultCredentials;

                // get the response on the request
                using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
                {
                    // we need to get the cookie
                    string cookieName = string.Empty;
                    string cookieValue = string.Empty;
                    const char separator = '=';
                    foreach (WebHeaderCollection collection in request.Headers)
                    {
                        if (collection.ToString() == "Cookie")
                        {
                            string value = request.Headers["Cookie"].ToString();
                            string[] keyvalues = value.Split(separator);
                            if (keyvalues.Length > 1)
                            {
                                cookieName = keyvalues[0];
                                // use cookieName +  separator value to split again in case we have more than one separator in Cookie value
                                string removeStr = cookieName + separator.ToString();
                                cookieValue = value.Replace(removeStr, string.Empty);
                                //what we need is only "Cookie" value from Headers
                                break;
                            }
                        }
                    }

                    // set the cookie for the formPost request
                    Cookie cookie = new Cookie();
                    cookie.Name = cookieName;
                    cookie.Value = cookieValue;
                    cookie.Domain = response.ResponseUri.Authority;
                    cookie.HttpOnly = true;
                    cookie.Secure = true;
                    cookies.Add(cookie);

                    // values and url to post on page
                    string postData = "agreementValue=Agree";
                    string redirectedTo = response.ResponseUri.AbsoluteUri;
                    redirectedTo = redirectedTo.Substring(0, redirectedTo.LastIndexOf("/"));
                    redirectedTo = redirectedTo + "/submitAgreement.do";

                    // post to the form on redirected page
                    HttpWebRequest formPostRequest = (HttpWebRequest)WebRequest.Create(redirectedTo);
                    formPostRequest.CookieContainer = cookies;
                    formPostRequest.Credentials = CredentialCache.DefaultCredentials;
                    formPostRequest.Method = WebRequestMethods.Http.Post;
                    formPostRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2";
                    formPostRequest.AllowWriteStreamBuffering = true;
                    formPostRequest.ProtocolVersion = HttpVersion.Version11;
                    formPostRequest.Referer = response.ResponseUri.AbsoluteUri;
                    formPostRequest.ContentType = "application/x-www-form-urlencoded";
                    byte[] byteArray = Encoding.ASCII.GetBytes(postData);
                    formPostRequest.ContentLength = byteArray.Length;
                    //open connection
                    Stream newStream = formPostRequest.GetRequestStream();
                    // Send the data.
                    newStream.Write(byteArray, 0, byteArray.Length);
                    // close it we don't need it anymore
                    newStream.Close();

                    // will return data .txt url data
                    using (HttpWebResponse formPostResponse = (HttpWebResponse)formPostRequest.GetResponse())
                    {
                        // Here we are going to read the contents of the downloaded file
                        // one line/record at a time. Each line/record defines a bank routing number.
                        // The format of the record is documented in the FedACHDirectoryRecord class.                       
                        using (StreamReader readStream = new StreamReader(formPostResponse.GetResponseStream()))
                        {
                            string text = null;
                            // Read all the lines out of the downloaded file stream.
                            // Each line defines a bank routing number (a banking institution).
                            while ((text = readStream.ReadLine()) != null)
                            {
                                // Wrap the line in a record instance (provides access to fields)
                                FedACHDirectoryRecord rec = new FedACHDirectoryRecord(text);
                                // Build the list, record by record
                                rnList.Add(rec);
                            }
                        }
                    }                    
                }