Click here to Skip to main content
15,869,940 members
Please Sign up or sign in to vote.
3.00/5 (2 votes)
See more:
I have to open a web page (https), enter username and password, click some links to get to a desired page and then download all files (pdf/txt) to a shared folder on our internal network.

I'd like to do this with windows application/Service which would run once in a day. What is the best way to do this?
Posted
Comments
MuhammadUSman1 20-Mar-13 23:42pm    
Hello Developers on(CP)
Any idea?

1 solution

See the example below, this will download any sourcecode attachwed to any article from CodeProject by logging in.

C#
private static void Download(string email, string password, string destination, string downloadUri)
{

    using (CookiesAwareWebClient client = new CookiesAwareWebClient())
    {
        NameValueCollection values = new NameValueCollection();
        values.Add("Email",email );
        values.Add("Password", password);
        values.Add("x", "10"); 
        values.Add("y", "10"); 
        values.Add("login", "login");

        // We authenticate first
        client.UploadValues("https://www.codeproject.com/script/Membership/LogOn.aspx?rp=%2f", values);

        // Now we can download

        string modifieduri = "http://www.codeproject.com/script/articles/download.aspx?file=" + downloadUri.Replace("http://www.codeproject.com","") + "&rp=";


        string filename = System.IO.Path.GetFileName(downloadUri);

        string filepathname = System.IO.Path.Combine(destination, filename);

        client.DownloadFile(modifieduri, filepathname);
    }
}



The class used(CookiesAwareWebClient) in the code above:
C#
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;

namespace CodeProjectDownload
{
    public class CookiesAwareWebClient : WebClient
    {
        public CookieContainer CookieContainer;
        public CookieContainer MyProperty
        {
            get { return CookieContainer; }
            set { CookieContainer = value; }
        }
        public CookiesAwareWebClient()
        {
            CookieContainer = new CookieContainer();
        }

        protected override WebRequest GetWebRequest(Uri address)
        {
            WebRequest request = base.GetWebRequest(address);
            ((HttpWebRequest)request).CookieContainer = CookieContainer;
            return request;
        }
    } 
}


Another Option(Generic):
C#
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace IndianaMedicAidAccess
{
    public class HTTHelper
    {
        private static HttpWebResponse POST(string postData, string url, string referer, string cookie)
        {
            try
            {
                byte[] byteArray = Encoding.UTF8.GetBytes(postData);
                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
                request.Method = "POST";
                request.KeepAlive = true;
                request.AllowAutoRedirect = false;
                request.Accept = "*/*";
                request.ContentType = "application/x-www-form-urlencoded";
                if (!string.IsNullOrEmpty(cookie))
                    request.Headers.Add(HttpRequestHeader.Cookie, cookie);
                if (!string.IsNullOrEmpty(referer))
                    request.Referer = referer;
                request.ContentLength = byteArray.Length;
                request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5";
                //   request.Proxy = null;
                Stream dataStream = request.GetRequestStream();
                dataStream.Write(byteArray, 0, byteArray.Length);
                dataStream.Close();
                try
                {
                    return (HttpWebResponse)request.GetResponse();
                }
                catch
                {
                    return (HttpWebResponse)null;
                }
            }
            catch
            {
                return (HttpWebResponse)null;
            }
        }
        public static string ProcessLogin(string postData, string LoginUrl, string Referer)
        {
            string LoginStatus = "Unsucessful";
            string uri = string.Empty;
            string cookie = string.Empty;
            HttpWebResponse response = POST(postData, LoginUrl, Referer, "");
            if (response == null)
            {
                return LoginStatus;
            }

            WebHeaderCollection headers = response.Headers;

            if ((response.StatusCode == HttpStatusCode.Found) ||
                    (response.StatusCode == HttpStatusCode.Redirect) ||
                    (response.StatusCode == HttpStatusCode.Moved) ||
                    (response.StatusCode == HttpStatusCode.MovedPermanently))
            {


                if (headers["Set-Cookie"] != null)
                {

                    string cookies = headers["Set-Cookie"];
                    List<string> fieldList = cookies.Split(',').ToList();
                    response.Close();
                    foreach (string str in fieldList)
                    {
                        try
                        {
                            if (str.IndexOf("=") > -1)
                            {
                                string mystr = str.Split(new string[] { ";" }, StringSplitOptions.RemoveEmptyEntries)[0];
                                cookie += mystr + ";";
                            }
                        }
                        catch { }
                    }
                    cookie = cookie.LastIndexOf(";") == cookie.Length - 1 ? cookie.Substring(0, cookie.Length - 1) : cookie;
                    if (string.IsNullOrEmpty(cookie))
                    {
                        String[] fields = Regex.Split(cookies, ";\\s*");
                        cookie = fields[0];
                       
                    }
                }
            }
            return cookie;

        }
    }
    class Program
    {
        static void Main(string[] args)
        {
            string LoginUrl = "https://interchange.indianamedicaid.com/Administrative/LogonValidate.aspx";
            string Referer="https://interchange.indianamedicaid.com/Administrative/logon.aspx?Logon=Y";
            string _userName = "youruserId"; // please change
            string _password = "passwd"; // please change
            string downloadUri = "https://xfile.indianamedicaid.com/......."; // please change
            string destination = Path.Combine("c:\\Somefolder", "FileName.ext");  //please change
            string postData = String.Format("logonid={1}&logonpswd={2}&Log On=Log On", _userName, _password);
            
            string cookie = HTTHelper.ProcessLogin(postData, LoginUrl, Referer);

            WebClient wb = new WebClient();
            wb.Headers.Add(HttpRequestHeader.Cookie, cookie);
            wb.DownloadFile(downloadUri, destination);
        }
    }
}
 
Share this answer
 
v3
Comments
bbirajdar 21-Mar-13 0:48am    
Good job +5.This should help
MuhammadUSman1 21-Mar-13 1:27am    
Thanks Dear.
it is working for code project. but on which website i am working there is no ID for UserName and Password and the Sign On i an image i am not logging in there. What i try there?
Kuthuparakkal 21-Mar-13 1:48am    
Please post the url you are trying to login/download
MuhammadUSman1 21-Mar-13 4:09am    
Using code project when i log in. I want to download latest 3 Articles on daily basis. tell how to get links and download selected Articles.

And thanks again Dear. when feel free please reply i am waiting for your Reply.
Kuthuparakkal 21-Mar-13 4:45am    
Updated solution with more generic idea, please check!

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)



CodeProject, 20 Bay Street, 11th Floor Toronto, Ontario, Canada M5J 2N8 +1 (416) 849-8900