HttpClient 大量连接等待异常的处理

前几天要扫一批连接，因为涉及到ua的适配测试，所以选用了httpclient处理。

httpclient其实很好用，也用过几次，还曾经写过自动刷人人网人气的程序，不过对于它掌握还是处在比较浅显的时代

我这人比较懒，学习动力还是太弱，不逼到份上都懒得学习哎！

这次又用到这东西了就直接看了看api 就开始写了，不过写的时候发现一个问题。

开始跑100多个url的时候是没问题的，不过过了一定量数后，会大量的抛出如下异常

org.apache.http.conn.ConnectionPoolTimeoutException: Timeout waiting for connection
at org.apache.http.impl.conn.tsccm.ConnPoolByRoute.getEntryBlocking(ConnPoolByRoute.java:417)
at org.apache.http.impl.conn.tsccm.ConnPoolByRoute$1.getPoolEntry(ConnPoolByRoute.java:300)
at org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager$1.getConnection(ThreadSafeClientConnManager.java:224)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:401)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:820)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:754)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:732)
at com.ua.engine.Engine.httpService(Engine.java:97)
at com.ua.engine.Engine.main(Engine.java:27)

对此很不解，还是了解的不够多啊，后来发现问题了，原来之前一直使用连接池中的一条连接进行链接访问，用的多了，连接到期，会造成连接等待的情况，

因为使用的HttpClient4 ,于是查了下资料，发现使用完每个连接后一定要及时的释放掉，不然上面的错会无限的报下去。

而释放连接只用InputStream 就能轻松搞定了

最后整体的代码如下

  1 package com.ua.engine;
  2 
  3 import java.io.BufferedReader;
  4 import java.io.BufferedWriter;
  5 import java.io.FileReader;
  6 import java.io.IOException;
  7 import java.io.InputStream;
  8 import java.io.PrintWriter;
  9 import java.util.ArrayList;
 10 import java.util.HashMap;
 11 import java.util.List;
 12 
 13 import org.apache.http.HttpEntity;
 14 import org.apache.http.HttpResponse;
 15 import org.apache.http.client.HttpClient;
 16 import org.apache.http.client.methods.HttpGet;
 17 
 18 
 19 public class Engine {
 20     
 21     public static void main(String[] args) {
 22         try {
 23             HashMap<String,List<String>> map = readURL();
 24             BufferedWriter bw = new BufferedWriter(new PrintWriter("D://furl/log.txt"));
 25                 List<String> weblist = map.get("web");
 26                 int i = 1;
 27                 for(String url :weblist){
 28                     System.out.println(i);
 29                     String temp = Engine.httpService("http://52.easou.com"+url,"web")+" "+url;
 30                     System.out.println(temp);
 31                     bw.write(temp);
 32                     bw.write("\r\n");
 33                     i++;
 34                 }
 35                 List<String> touchlist = map.get("touch");
 36                 for(String url :touchlist){
 37                     System.out.println(i);
 38                     String temp = Engine.httpService("http://52.easou.com"+url,"touch")+" "+url;
 39                     System.out.println(temp);
 40                     bw.write(temp);
 41                     bw.write("\r\n");
 42                     i++;
 43                 }
 44                 List<String> hdlist = map.get("hd");
 45                 for(String url :hdlist){
 46                     System.out.println(i);
 47                     String temp = Engine.httpService("http://52.easou.com"+url,"hd")+" "+url;
 48                     System.out.println(temp);
 49                     bw.write(temp);
 50                     bw.write("\r\n");
 51                     i++;
 52                 }
 53                 List<String> cslist = map.get("cs");
 54                 for(String url :cslist){
 55                     System.out.println(i);
 56                     String temp = Engine.httpService("http://52.easou.com"+url,"cs")+" "+url;
 57                     System.out.println(temp);
 58                     bw.write(temp);
 59                     bw.write("\r\n");
 60                     i++;
 61                 }
 62                 bw.flush();
 63                 bw.close();
 64         }
 65         catch (Exception e) {
 66             e.printStackTrace();
 67         }
 68     }
 69     private static HttpClient httpclient = null;
 70     static {
 71         HttpClientUtils hcu = new HttpClientUtils();
 72         hcu.setConnTimeout(10000);
 73         hcu.setSoTimeout(10000);
 74         hcu.setBufSize(128 * 1024);
 75         hcu.setAgent("");
 76         hcu.setMaxRedirect(3);
 77         hcu.setMaxTotal(500);
 78         hcu.setMaxPerRoute(100);
 79         httpclient = hcu.newClient();
 80     }
 81     public static String httpService(String url,String version) {
 82         HttpGet get = new HttpGet(url);
 83         if("web".equals(version)){
 84             get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.52 Safari/536.5");
 85         }else if("touch".equals(version)){
 86             get.setHeader("User-Agent", "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16");
 87         }else if("hd".equals(version)){
 88             get.setHeader("User-Agent", "Mozilla/5.0 (iPad; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16");
 89         }else {
 90             get.setHeader("User-Agent", "lg-kp500 teleca/wap2.0 midp-2.0/cldc-1.1 untrusted/1.0 nokia7610 (3.0417.0ch) symbianos/7.0s series60/2.1 profile/midp-2.0 conf");
 91         }
 92 
 93 //        HttpEntity entity = response.getEntity();
 94 //        System.out.println(EntityUtils.toString(entity));
 95         HttpResponse response = null;
 96         String code = "error "+url;
 97         InputStream in = null;  
 98         try {
 99             response = httpclient.execute(get);
100             HttpEntity entity =response.getEntity();  
101             if( entity != null ){   
102              in = entity.getContent(); //之前没使用这个造成了大量异常抛出，只要是
103             }
104             code = response.getStatusLine().getStatusCode()+" "+version+" "+url;
105             return code;
106         }
107         catch (Exception e) {
108             System.out.println(code);
109             e.printStackTrace();
110         }finally {
111             if(in!=null){
112                 try {
113                     in.close();
114                 }
115                 catch (IOException e) {
116                     e.printStackTrace();
117                 }
118             }
119         }
120         return code;
121     }
122     
123     public static HashMap<String,List<String>> readURL() throws Exception{
124         List<String> weblist = new ArrayList<String>();
125         List<String> touchlist = new ArrayList<String>();
126         List<String> hdlist = new ArrayList<String>();
127         List<String> cslist = new ArrayList<String>();
128         
129         BufferedReader br = new BufferedReader(new FileReader("D://furl/1.txt"));
130         while(br.ready()){
131             String url = br.readLine();
132             if(url.toLowerCase().indexOf(".html")>-1){
133                 weblist.add(url);
134             }else if(url.toLowerCase().indexOf("touch")>-1){
135                 touchlist.add(url);
136             }else if(url.toLowerCase().indexOf("/hd/")>-1){
137                 hdlist.add(url);
138             }else {
139                 cslist.add(url);
140             }
141         }
142         br = new BufferedReader(new FileReader("D://furl/url1.txt"));
143         while(br.ready()){
144             String url = br.readLine();
145             if(url.toLowerCase().indexOf(".html")>-1){
146                 weblist.add(url);
147             }else if(url.toLowerCase().indexOf("touch")>-1){
148                 touchlist.add(url);
149             }else if(url.toLowerCase().indexOf("/hd/")>-1){
150                 hdlist.add(url);
151             }else {
152                 cslist.add(url);
153             }
154         }
155         br = new BufferedReader(new FileReader("D://furl/urljian1.txt"));
156         while(br.ready()){
157             String url = br.readLine();
158             if(url.toLowerCase().indexOf(".html")>-1){
159                 weblist.add(url);
160             }else if(url.toLowerCase().indexOf("touch")>-1){
161                 touchlist.add(url);
162             }else if(url.toLowerCase().indexOf("/hd/")>-1){
163                 hdlist.add(url);
164             }else {
165                 cslist.add(url);
166             }
167         }
168         HashMap<String,List<String>> map = new HashMap<String,List<String>>();
169         System.out.println(weblist.size());
170         System.out.println(touchlist.size());
171         System.out.println(hdlist.size());
172         System.out.println(cslist.size());
173         map.put("web", weblist);
174         map.put("touch", touchlist);
175         map.put("hd", hdlist);
176         map.put("cs", cslist);
177         return map;
178     }
179 }

 1 package com.ua.engine;
 2 
 3 import org.apache.http.HttpVersion;
 4 import org.apache.http.client.HttpClient;
 5 import org.apache.http.client.params.ClientPNames;
 6 import org.apache.http.impl.client.DefaultHttpClient;
 7 import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
 8 import org.apache.http.params.BasicHttpParams;
 9 import org.apache.http.params.CoreConnectionPNames;
10 import org.apache.http.params.CoreProtocolPNames;
11 import org.apache.http.params.HttpParams;
12 
13 public class HttpClientUtils {
14     private int connTimeout = 5000;
15     private int soTimeout = 5000;
16     private int bufSize = 128 * 1024;
17     private String agent = "";
18     private int maxRedirect = 3;
19     private int maxTotal = 500;
20     private int maxPerRoute = 100;
21 
22     public HttpClient newClient() {
23         HttpParams params = new BasicHttpParams();
24         params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,
25                 this.connTimeout);
26         params.setParameter(CoreConnectionPNames.SO_TIMEOUT, this.soTimeout);
27         params.setParameter(CoreConnectionPNames.SO_REUSEADDR, true);
28         params.setParameter(CoreConnectionPNames.TCP_NODELAY, false);
29         params.setParameter(CoreConnectionPNames.SOCKET_BUFFER_SIZE,
30                 this.bufSize);
31         params.setParameter(CoreProtocolPNames.PROTOCOL_VERSION,
32                 HttpVersion.HTTP_1_1);
33         params.setParameter(CoreProtocolPNames.USER_AGENT, this.agent);
34         params.setParameter(ClientPNames.CONNECTION_MANAGER_FACTORY_CLASS_NAME,
35                 "org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager");
36         params.setParameter(ClientPNames.MAX_REDIRECTS, this.maxRedirect);
37         ThreadSafeClientConnManager ccm = new ThreadSafeClientConnManager();
38         ccm.setMaxTotal(this.maxTotal);
39         ccm.setDefaultMaxPerRoute(this.maxPerRoute);
40         return new DefaultHttpClient(ccm, params);
41     }
42 
43     public long getConnTimeout() {
44         return connTimeout;
45     }
46 
47     public void setConnTimeout(int connTimeout) {
48         this.connTimeout = connTimeout;
49     }
50 
51     public long getSoTimeout() {
52         return soTimeout;
53     }
54 
55     public void setSoTimeout(int soTimeout) {
56         this.soTimeout = soTimeout;
57     }
58 
59     public int getBufSize() {
60         return bufSize;
61     }
62 
63     public void setBufSize(int bufSize) {
64         this.bufSize = bufSize;
65     }
66 
67     public String getAgent() {
68         return agent;
69     }
70 
71     public void setAgent(String agent) {
72         this.agent = agent;
73     }
74 
75     public int getMaxRedirect() {
76         return maxRedirect;
77     }
78 
79     public void setMaxRedirect(int maxRedirect) {
80         this.maxRedirect = maxRedirect;
81     }
82 
83     public int getMaxTotal() {
84         return maxTotal;
85     }
86 
87     public void setMaxTotal(int maxTotal) {
88         this.maxTotal = maxTotal;
89     }
90 
91     public int getMaxPerRoute() {
92         return maxPerRoute;
93     }
94 
95     public void setMaxPerRoute(int maxPerRoute) {
96         this.maxPerRoute = maxPerRoute;
97     }
98 }