从txt文本中读取数据，并发送请求
public static void main(String[] args) {
String pathname = "C:\Users\ceshi\Desktop\test1.txt";
try{
FileReader reader = new FileReader(pathname);
BufferedReader br = new BufferedReader(reader);
String line;
while ((line = br.readLine()) != null) {
// 一次读入一行数据
String s = WebResourceReader.doGet(line);
System.out.println(s);
}
}catch (Exception e){
e.printStackTrace();
}
}
发送请求工具类
package com.ceshi.util;
import com.alibaba.fastjson.JSONArray;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Map;
import java.util.zip.GZIPInputStream;
public class WebResourceReader {
private static final Logger log = LoggerFactory.getLogger(WebResourceReader.class);
protected static final Logger STDERR = LoggerFactory.getLogger("STDERR");
private static String encoding = "gbk";
private static String getHost(String url){
String regexp = "(http://)?([^/]*)(/?.*)";
return url.replaceAll(regexp, "$2");
}
private static String getPath(String url){
return url.replaceAll("http://"+getHost(url), "");
}
public static String read(String url) {
StringBuffer buffer = new StringBuffer();
BufferedReader in = null;
GZIPInputStream gzin = null;
GetMethod getMethod = null;
try {
PostUrl pu = new PostUrl();
pu.setHost(getHost(url));
pu.setPath(getPath(url));
pu.setPort(80);
// 创建GET方法的实例
getMethod = HttpClientUtil.getInstance().get(pu);
// 使用系统提供的默认的恢复策略
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler());
// 执行getMethod
int statusCode = getMethod.getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
log.error("读取远程资源文件失败: "+ url);
}
InputStream is = getMethod.getResponseBodyAsStream();
String contEncoding ="";
Header head = getMethod.getResponseHeader(
"Content-Encoding");
if(null!=head){
contEncoding = head.getValue();
}
/**
* 如果文件使用GZIP压缩，则用GZIP流进行处理
*/
if (StringUtils.isNotBlank(contEncoding)
&& contEncoding.contains("gzip")) {
// 读取内容
gzin = new GZIPInputStream(is);
in = new BufferedReader(new InputStreamReader(gzin, encoding));
} else {
in = new BufferedReader(new InputStreamReader(is, encoding));
}
String inputLine;
while ((inputLine = in.readLine()) != null) {
buffer.append(inputLine);
}
}catch (Exception e) {
log.error("##read error=",e);
} finally {
// 释放连接
if(getMethod!=null){
getMethod.releaseConnection();
}
try {
if(in!=null)
in.close();
} catch (IOException e) {
log.error("Read remote file exception : ",e);
}
}
return buffer.toString();
}
/**
* HttpClientUtil get方法
* @param url
* @return
* @throws Exception
*/
public static String getHttpResponse(String url) throws Exception{
BufferedReader in = null;
InputStream is = null;
GetMethod getMethod = null;
try {
PostUrl pu = new PostUrl();
pu.setHost(getHost(url));
pu.setPath(getPath(url));
pu.setPort(80);
getMethod = HttpClientUtil.getInstance().get(pu);
getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,new DefaultHttpMethodRetryHandler());
int statusCode = getMethod.getStatusCode();
log.info("##getHttpResponse-url="+url+"|http-status="+ statusCode);
is = getMethod.getResponseBodyAsStream();
StringBuffer buffer = new StringBuffer();
in = new BufferedReader(new InputStreamReader(is, "utf-8"));
String inputLine;
while ((inputLine = in.readLine()) != null) {
buffer.append(inputLine);
}
return buffer.toString();
} catch (Exception e) {
STDERR.error("##getHttpResponse error=",e);
throw e;
}finally{
try {
if(is!=null)
is.close();
if(in!=null)
in.close();
if(getMethod!=null)
getMethod.releaseConnection();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* HttpURLConnection Post方法
* @param url
* @param param
* @return
* @throws Exception
*/
public static String doPost(String url, String param) throws Exception {
URL url1 = null;
BufferedReader reader = null;
PrintWriter writer = null;
HttpURLConnection conn = null;
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("POST");
conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)");
conn.setDoInput(true);
conn.setDoOutput(true);
writer = new PrintWriter(conn.getOutputStream());
writer.print(param);
writer.flush();
int resCode = conn.getResponseCode();
log.info("##doPost……url="+url+",param="+param+", Response code is " + resCode);
if(resCode==200)
{
reader = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##HTTP Request is error,url="+url+",param="+param+", error=",e);
throw e;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doPost error1:",e1);
}
}
if (writer != null) {
try {
writer.close();
} catch (Exception e) {
STDERR.error("##doPost error2:",e);
}
}
if (conn != null)
try {
conn.disconnect();
} catch (Exception e) {
STDERR.error("##doPost error3:",e);
}
}
}
/**
* headers的value暂时支持1个参数
* @param url
* @param param
* @param headers
* @return
*/
public static String doPost(String url, String param, HttpHeaders headers) throws Exception {
Iterator iterator = headers.entrySet().iterator();
HashMap<Object, Object> map = new HashMap<Object, Object>();
while (iterator.hasNext()) {
HttpHeaders.Entry entry = (HttpHeaders.Entry) iterator.next();
Object key = entry.getKey();
Object vals = entry.getValue();
if (vals instanceof LinkedList) {
String val = (String) ((LinkedList) vals).get(0);
if (val instanceof String && org.apache.commons.lang3.StringUtils.isNotBlank(val)) {
map.put(key, val);
}
}
}
return doPost(url, param, map);
}
/**
* HttpURLConnection Post方法，待测
* @param url
* @param param
* @return
* @throws Exception
*/
public static String doPost(String url, String param, Map headers) throws Exception {
URL url1 = null;
BufferedReader reader = null;
DataOutputStream writer = null;
GZIPInputStream gzin = null;
HttpURLConnection conn = null;
BufferedReader input = null;
String encodeCharset = "UTF-8";
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("POST");
conn.setInstanceFollowRedirects(true);
Iterator iterator = headers.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry entry = (Map.Entry) iterator.next();
Object key = entry.getKey();
Object val = entry.getValue();
if (key instanceof String && val instanceof String)
conn.setRequestProperty((String) key, (String) val);
}
conn.setRequestProperty("Accept-Charset", "UTF-8");
conn.setRequestProperty("contentType", "UTF-8");
conn.setDoInput(true);
conn.setDoOutput(true);
writer = new DataOutputStream(conn.getOutputStream());
writer.write(param.getBytes());
writer.flush();
writer.close();
int resCode = conn.getResponseCode();
log.info("##doPost_header_url=" + url + ",param=" + param + ", Response code is " + resCode);
if (resCode == 200) {
String acceptEncoding = conn.getRequestProperty("accept-encoding");
/**
* 如果文件使用GZIP压缩，则用GZIP流进行处理
*/
if (StringUtils.isNotBlank(acceptEncoding)
&& acceptEncoding.contains("gzip")) {
// 读取内容
gzin = new GZIPInputStream(conn.getInputStream());
input = new BufferedReader(new InputStreamReader(gzin,encodeCharset));
} else {
input = new BufferedReader(new InputStreamReader(conn.getInputStream(),encodeCharset));
}
String result = "";
String str;
while (null != (str = input.readLine())) {
result += str;
}
return result;
}
return "";
} catch (Exception e) {
e.printStackTrace();
STDERR.error("##HTTP Request is error,url=" + url + ",param=" + param + ", error=", e);
throw e;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doPost error1:", e1);
}
}
if (writer != null) {
try {
writer.close();
} catch (Exception e) {
STDERR.error("##doPost error2:", e);
}
}
if (conn != null)
try {
conn.disconnect();
} catch (Exception e) {
STDERR.error("##doPost error3:", e);
}
}
}
/**
* HttpURLConnection get方法
* @param url
* @return
*/
public static String doGet(String url) {
URL url1 = null;
BufferedReader reader = null;
HttpURLConnection conn = null;
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(true);
conn.connect();
if (conn.getResponseCode()== 200)
{
reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##doGet error,url="+url+", error=",e);
throw new RuntimeException(e.getMessage(),e);
} finally {
try {
if(reader!=null)reader.close();
if(conn!=null)conn.disconnect();
} catch (Exception e) {
STDERR.error("##doGet finally error,url="+url+", error=",e);
e.printStackTrace();
}
}
}
/**
* HttpURLConnection PUT方法
* @param url
* @return
*/
public static String doPUT(String url,Map<String,String> headers) {
URL url1 = null;
BufferedReader reader = null;
HttpURLConnection conn = null;
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("PUT");
if (headers != null && headers.size() > 0) {
for (Map.Entry<String, String> entry : headers.entrySet()) {
conn.setRequestProperty(entry.getKey(), entry.getValue());
}
}
conn.getOutputStream().flush();
conn.getOutputStream().close();
conn.connect();
int code = conn.getResponseCode();
if (code == 200) {
reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
log.info("##doPut code" + code);
return "";
} catch (IOException e) {
STDERR.error("##doPut error,url="+url+", error=",e);
throw new RuntimeException(e.getMessage(),e);
} finally {
try {
if(reader!=null)reader.close();
if(conn!=null)conn.disconnect();
} catch (Exception e) {
STDERR.error("##doPut error1,url="+url+", error=",e);
}
}
}
/**
* HttpURLConnection get方法
* @param url
* @return
*/
public static String doGet(String url, String userAgent, Boolean useUserAgent,String clientIp) {
URL url1 = null;
BufferedReader reader = null;
HttpURLConnection conn = null;
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(true);
if(useUserAgent) {
conn.addRequestProperty("User-Agent", userAgent);
}
if(StringUtils.isNotBlank(clientIp)){
conn.addRequestProperty("j-forwarded-for",clientIp);
conn.addRequestProperty("x-forwarded-for",clientIp);
}
conn.connect();
if (conn.getResponseCode()== 200)
{
reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##doGet error,url="+url+", error=",e);
throw new RuntimeException(e.getMessage(),e);
} finally {
try {
if(reader!=null)reader.close();
if(conn!=null)conn.disconnect();
} catch (Exception e) {
STDERR.error("##doGet finally error,url="+url+", error=",e);
e.printStackTrace();
}
}
}
/**
* HttpURLConnection get方法
* @param url
* @return
*/
public static String doGet(String url,String charset) {
URL url1 = null;
BufferedReader reader = null;
HttpURLConnection conn = null;
if(StringUtils.isBlank(charset)){
charset = "utf-8";
}
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("GET");
conn.setInstanceFollowRedirects(true);
conn.connect();
if (conn.getResponseCode()== 200)
{
reader = new BufferedReader(new InputStreamReader(conn.getInputStream(),charset));
StringBuilder sb = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##doGet error,url="+url+", error=",e);
throw new RuntimeException(e.getMessage(),e);
} finally {
try {
if(reader!=null)reader.close();
if(conn!=null)conn.disconnect();
} catch (Exception e) {
STDERR.error("##doGet finally error,url="+url+", error=",e);
e.printStackTrace();
}
}
}
/**
* HttpURLConnection Post方法获取请求
* @param url
* @param param
* @return
*/
public static Map<String,Object> doPostForLive(String url, String param){
URL url1 = null;
BufferedReader reader = null;
PrintWriter writer = null;
HttpURLConnection connection = null;
Map<String,Object> map = new HashMap<String,Object>();
try {
url1 = new URL(url);
connection = (HttpURLConnection) url1.openConnection();
connection.setConnectTimeout(20000);
connection.setReadTimeout(20000);
connection.setRequestMethod("POST");
connection.setInstanceFollowRedirects(true);
connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)");
connection.setDoInput(true);
connection.setDoOutput(true);
writer = new PrintWriter(connection.getOutputStream());
writer.print(param);
writer.flush();
int resCode = connection.getResponseCode();
map.put("rescode", resCode);
log.info("##doPostForLive……url="+url+",param="+param+", Response code is " + resCode);
if(resCode!= HttpStatus.SC_OK){
reader = new BufferedReader(new InputStreamReader(connection.getErrorStream()));
}else{
reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
}
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
map.put("resmess", sb.toString());
} catch (Exception e) {
STDERR.error("##doPostForLive Request is error,url="+url+",param="+param+", error=",e);
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doPostForLive error1:",e1);
}
}
if (writer != null) {
try {
writer.close();
} catch (Exception e) {
STDERR.error("##doPostForLive error2:",e);
}
}
if (connection != null)
try {
connection.disconnect();
} catch (Exception e) {
STDERR.error("##doPostForLive error3:",e);
}
}
return map;
}
/**
* HttpURLConnection Get方法获取请求
* @param url
* @return
*/
public static Map<String,Object> doGetForLive(String url){
URL url1 = null;
BufferedReader reader = null;
HttpURLConnection connection = null;
Map<String,Object> map = new HashMap<String,Object>();
try {
url1 = new URL(url);
connection = (HttpURLConnection) url1.openConnection();
connection.setConnectTimeout(20000);//设置连接主机超时（单位：毫秒）
connection.setReadTimeout(20000);//设置从主机读取数据超时（单位：毫秒）
connection.setRequestMethod("GET");
connection.setInstanceFollowRedirects(true);
connection.connect();
int resCode = connection.getResponseCode();
log.info("##doGetLive……url="+url+", Response code is " + resCode);
map.put("rescode", resCode);
if(resCode!= HttpStatus.SC_OK){
reader = new BufferedReader(new InputStreamReader(connection.getErrorStream(),"UTF-8"));
}else{
reader = new BufferedReader(new InputStreamReader(connection.getInputStream(),"UTF-8"));
}
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
map.put("resmess", sb.toString());
} catch (Exception e) {
STDERR.error("##doGetLive Request is error,url="+url+", error=",e);
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doGetLive error1:",e1);
}
}
if (connection != null)
try {
connection.disconnect();
} catch (Exception e) {
STDERR.error("##doGetLive error3:",e);
}
}
return map;
}
/**
* 抽奖
* @param url
* @param param
* @return
* @throws Exception
*/
public static String doPostForLottery(String url, String param) throws Exception {
URL url1 = null;
BufferedReader reader = null;
PrintWriter writer = null;
HttpURLConnection connection = null;
try {
url1 = new URL(url);
connection = (HttpURLConnection) url1.openConnection();
connection.setConnectTimeout(20000);
connection.setReadTimeout(20000);
connection.setRequestMethod("POST");
connection.setInstanceFollowRedirects(true);
connection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)");
connection.setRequestProperty("referer", "http://live.jd.com");
connection.setDoInput(true);
connection.setDoOutput(true);
writer = new PrintWriter(connection.getOutputStream());
writer.print(param);
writer.flush();
int resCode = connection.getResponseCode();
log.info("##doPostForLottery……url="+url+",param="+param+", Response code is " + resCode);
if(resCode==200)
{
reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##doPostForLottery HTTP Request is error,url="+url+",param="+param+", error=",e);
throw e;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doPostForLottery error1:",e1);
}
}
if (writer != null) {
try {
writer.close();
} catch (Exception e) {
STDERR.error("##doPostForLottery error2:",e);
}
}
if (connection != null)
try {
connection.disconnect();
} catch (Exception e) {
STDERR.error("##doPostForLottery error3:",e);
}
}
}
/**
* 获取外部数据 try3次
*
* @param url
* @return
*/
public static String getDataTry(String url, String param, String type) {
if (StringUtils.isBlank(url)) {
return null;
}
log.info("##getDataTry----url=" + url + "|param=" + param + "|type="+ type);
String data = StringUtils.EMPTY;
// 失败尝试3次
int try_num = 3;
for (int retryTime = 1; retryTime <= try_num; retryTime++) {
try {
if (StringUtils.isNotBlank(type) && "POST".equals(type)) {
data = WebResourceReader.doPost(url, param);
} else if (StringUtils.isNotBlank(type) && "GET".equals(type)) {
data = WebResourceReader.doGet(url);
}
break;
} catch (Exception e) {
if (retryTime == try_num) {
STDERR.error("##getDataTry...url" + url + "|try" + try_num
+ "次 fail |param=" + param + "|type=" + type
+ "|error=", e);
} else {
log.info("##getDataTry...url=" + url + "|try="
+ retryTime);
continue;
}
}
}
log.info("##getDataTry...url=" + url + "|param=" + param + "|type="
+ type + "|data=" + data);
return data;
}
/**
* HttpURLConnection Post方法 参数json格式
* @param url
* @param param
* @return
* @throws Exception
*/
public static String doPostJson(String url, String param) throws Exception {
URL url1 = null;
BufferedReader reader = null;
PrintWriter writer = null;
HttpURLConnection conn = null;
try {
url1 = new URL(url);
conn = (HttpURLConnection) url1.openConnection();
conn.setConnectTimeout(20000);
conn.setReadTimeout(20000);
conn.setRequestMethod("POST");
conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("accept", "*/*");
conn.setRequestProperty("connection", "Keep-Alive");
conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0)");
conn.setRequestProperty("Content-Type", "application/json;charset=UTF-8");
conn.setDoInput(true);
conn.setDoOutput(true);
writer = new PrintWriter(conn.getOutputStream());
writer.print(param);
writer.flush();
int resCode = conn.getResponseCode();
log.info("##doPost……url="+url+",param="+param+", Response code is " + resCode);
if(resCode==200)
{
reader = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line).append("
");
}
return sb.toString();
}
return "";
} catch (IOException e) {
STDERR.error("##HTTP Request is error,url="+url+",param="+param+", error=",e);
throw e;
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e1) {
STDERR.error("##doPost error1:",e1);
}
}
if (writer != null) {
try {
writer.close();
} catch (Exception e) {
STDERR.error("##doPost error2:",e);
}
}
if (conn != null)
try {
conn.disconnect();
} catch (Exception e) {
STDERR.error("##doPost error3:",e);
}
}
}
}