use java thread to download large number of web pages

See more:

Java

Hello, I face a problem when I crawl lots of web pages(there are more than 50,000 url to process), it excute very slow,so I want to refactor it with thread,can anybody give some idea,thank you very much!

import ......
public class Down2011CaseMeshTread extends Thread {
	
    public static int count = 0;
    public static List<String> docDoiList = getCaseDoiList2();
    private static URL url;  
    private static String doi;
    static int BUFFER_SIZE = 1024*10;
    
    public Down2011CaseMeshTread(String doi) throws MalformedURLException{
    	String urlStr = "http://www.codeproject.com/" + doi;
    	this.url = new URL(urlStr);
    	this.doi = doi;
    }
    
    public static Connection getConnection() throws Exception {
        String driver = "com.mysql.jdbc.Driver";
        String url = "jdbc:mysql://192.168.1.102:3306/clef11";
        String username = "root";
        String password = "111111";
        Class.forName(driver);
        Connection conn = DriverManager.getConnection(url, username, password);
        return conn;
      }
    
    public static String getDocNameByDoi(String docDoi){
        ResultSet rs = null;
        Connection conn = null;
        PreparedStatement pstmt = null;
        String docName = null;
        try {
          conn = getConnection();
          String query = "select filename from casebase where doi = ?";
          pstmt = conn.prepareStatement(query); // create a statement
          pstmt.setString(1, docDoi); // set input parameter
          rs = pstmt.executeQuery();
          while(rs.next()){
        	  String name = rs.getNString(1);
        	  docName = name.subSequence(0, name.lastIndexOf(".")).toString();
          }
        } catch (Exception e) {
          e.printStackTrace();
        } finally {
          try {
            rs.close();
            pstmt.close();
            conn.close();
          } catch (SQLException e) {
            e.printStackTrace();
          }
        }
		return docName;
    }
    
	public void Test() throws IOException, InterruptedException{
		
		StringBuffer sb = null;
		BufferedReader in = null;
		BufferedWriter out = null;
		try {
			 sb = new StringBuffer();
			 int ch =0;
			 URLConnection conn = (HttpURLConnection)url.openConnection();
			 conn.setRequestProperty("User-Agent", "Mozilla/4.76 (compatible; MSIE 5.0; Windows NT; DigExt)");
			 conn.setDoOutput(true);
			 conn.setConnectTimeout(1000*60*10);
			 in = new BufferedReader(new InputStreamReader(url.openStream()));
		     FileOutputStream fo = new FileOutputStream("/home/boge/workspace1/IDF/case_mesh/" + getDocNameByDoi(doi));
			 OutputStreamWriter writer = new OutputStreamWriter(fo, "utf-8");
			 out = new BufferedWriter(writer);
			 while (!in.ready())
			 {
			      Thread.sleep(500); // wait for stream to be ready.
			 }
			 char[] buffer = new char[BUFFER_SIZE]; 
	                 int charsRead = 0;
	                 while ( (charsRead = in.read(buffer, 0, BUFFER_SIZE)) != -1 ) {
	                 out.write(buffer, 0, charsRead);
	         }
		 out.close();
		 in.close();
	    }catch(Exception e){
	    	e.printStackTrace();
	    }
	}
	
	public void run(){
		try {
			Test();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
	
	public static List<String> getCaseDoiList2(){
		List<String> docDoiList = new LinkedList<String>();
		BufferedReader br = null;
		try{
			br = new BufferedReader(new InputStreamReader(new FileInputStream("/home/boge/5.28.3")));
			String line = null;
			while((line = br.readLine())!=null){
				docDoiList.add(line.trim());
			}
		}catch(Exception e){
			e.printStackTrace();
		}
		return docDoiList;
	}
	
	public static List<String> getFiles(String fileName) {
		List<String> fileList = new ArrayList<String>();
               File directory = new File(fileName);
               for (File file : directory.listFiles()) {
        	  if (file.isFile() && !file.isHidden()) {
        		fileList.add(file.getName());
        	}
        }
        return fileList;	
	}
	
	public static void main(String args[]) throws MalformedURLException{
		List<String> filedowns = getFiles("/home/boge/workspace1/IDF/case_mesh");
		for(String docDoi : docDoiList){
			if(filedowns.contains(getDocNameByDoi(docDoi))){
				continue;
			}else{
				Down2011CaseMeshTread down = new Down2011CaseMeshTread(docDoi);
				down.start();
			}
		}
	}
}

But this code can work correctly, can you help me! :)