如有侵权，请联系作者删除
水平有限，还望大牛指点
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.8.3</version>
</dependency>
import com.sun.tools.doclets.formats.html.SourceToHTMLConverter; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * Created with Chenquan. * Description: 淘宝抓包 * Date: 2018-12-13 * Time: 15:12 */ public class TaobaoCatch { public static void main(String[] args) { int i = 0; /* String url = "https://acs.m.taobao.com/h5/mtop.taobao.wsearch.h5search/1.0/?jsv=2.3.16&appKey=12574478&t=1545023581359&sign=e3476c9041a75de0a9190da470204d93&api=mtop.taobao.wsearch.h5search&v=1.0&H5Request=true&ecode=1&type=jsonp&dataType=jsonp&callback=mtopjsonp1&data=%7B%22q%22%3A%22%E4%BB%99%E6%B6%B5%E5%86%85%E8%A1%A3%22%2C%22search%22%3A%22%E6%8F%90%E4%BA%A4%22%2C%22tab%22%3A%22all%22%2C%22sst%22%3A%221%22%2C%22n%22%3A20%2C%22buying%22%3A%22buyitnow%22%2C%22m%22%3A%22api4h5%22%2C%22token4h5%22%3A%22%22%2C%22abtest%22%3A%221%22%2C%22wlsort%22%3A%221%22%2C%22page%22%3A1%7D"; Connection con = Jsoup.connect(url); con.header("Cookie", "cna=TA+aFFGXQFUCAXQaRYGZVU8Q; t=efa81a9785cd86f885e13998b6d5f9cb; thw=cn; uc3=vt3=F8dByRzMU9X8Hvccr00%3D&id2=W8zLpWipxVFu&nk2=0PLo6GHZOM8%3D&lg2=V32FPkk%2Fw0dUvg%3D%3D; tracknick=%5Cu9648%5Cu94E81992; lgc=%5Cu9648%5Cu94E81992; _cc_=Vq8l%2BKCLiw%3D%3D; tg=0; enc=4rB%2FfKFx8DJKgPpoHlZjr824CEYw%2BlPaKBDWbFO4fnh6svGA97NoZNGERui4fOo2tXSnSVN1ygkfn5R5ekztTQ%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; mt=ci=0_1; _m_h5_tk=e501ac7690832934d663aef19ee36be5_1545033419107; _m_h5_tk_enc=5147579a652b4fb508dc886d59c37045; isg=BFVVgDOkpYNz64H7Z31pC9thZFHP-goqhI4h7tf6EUwbLnUgn6IZNGPv_DSYLiEc"); // con.header("referer", "https://item.taobao.com/item.htm "); Connection.Response resp = null; try { resp = con.method(Connection.Method.GET).ignoreContentType(true).execute(); } catch (IOException e) { e.printStackTrace(); } String body = resp.body(); // System.out.println(body); body = body.substring(12, body.length() - 1); JSONObject jb = JSONObject.fromObject(body); JSONArray jsonArray = jb.getJSONObject("data").getJSONArray("listItem"); // while(i<100){ i++; for (int j = 0; j < jsonArray.size(); j++) { JSONObject jsonObject = jsonArray.getJSONObject(j); String item_id = jsonObject.getString("item_id"); System.out.println("item_id: "+item_id); getAll(item_id); } // }*/ //传链接上的产品id getAll("577996531297"); } public static void getAll(String item_id ) { try { Thread.sleep(2000);//一个休息5s，太快会被禁 } catch (InterruptedException e) { e.printStackTrace(); } System.out.println("开始时间：" + new Date()); Date dateStart = new Date(); Document doc = null; String id = ""; try { // int i = 0; // while(i < 10000){ // i++; String url = "https://item.taobao.com/item.htm?id="+item_id; id = getParamByUrl(url, "id"); doc = Jsoup.connect(url).ignoreContentType(true).get(); /* String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格 String enc = "{"itemNumId":"582061497975"}"; String gbk = URLEncoder.encode(enc, "utf-8"); String sds = url + gbk; System.out.println("库存、价格"+sds); doc = Jsoup.connect(sds).ignoreContentType(true).get();*/ //设置请求头 // Connection con = Jsoup.connect(url); // con.header("Cookie", " enc=1LWJWtPGgf6MF1NVsn2rbeRb3%2FU1%2Fk5ZiiedHbVedmxmfvUUWDPmFeyKeLYl7NVchBB19JCIVnX0eFv4otK9HA%3D%3D;" + // "x5sec=7b2264657461696c736b69703b32223a226235653133353933646637396131353230343663346139633633653038326465434c6a4e7a654146454e447739724732716644534b426f4c4f4455774d7a51304e7a4d794f7a453d227d;" ); // con.header("referer", "https://item.taobao.com/item.htm "); // Connection.Response resp=con.method(Connection.Method.GET).execute(); // Map<String,String> cookies = resp.cookies(); // Connection.Request request = con.request(); // String body = resp.body(); } catch (IOException e) { e.printStackTrace(); } if (doc.baseUri().contains("tmall")) { System.out.println("商品名称："+ doc.select("h1[data-spm="1000983"]").text()); }else { System.out.println("商品名称：" + doc.select("h3[class="tb-main-title"]").text()); } Elements imgSrcElement = doc.select("#J_UlThumb > li"); for (Element element : imgSrcElement) { String imgSrc = ""; if (element.baseUri().contains("tmall")){ imgSrc = element.getElementsByTag("img").attr("src"); }else{ imgSrc = element.getElementsByTag("img").attr("data-src"); } imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/", ""); imgSrc = imgSrc.substring(0, imgSrc.length() - 10); // imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据 System.out.println("主图url：" + imgSrc); } // 规格参数 Elements selectRules = doc.select(".J_TSaleProp"); List<List<String>> liHashMap = new ArrayList<>(); for (Element ulElement : selectRules) { String ul = ulElement.getElementsByTag("ul").attr("data-property"); System.out.println("ul：" + ul); List<String> liString = new ArrayList<>(); for (Element liElement : ulElement.getElementsByTag("li")) { String liDataValue = liElement.getElementsByTag("li").attr("data-value"); System.out.println("liDataValue: " + liDataValue); liString.add(liDataValue); String aStyle = liElement.getElementsByTag("a").attr("style"); if (StringUtils.isNotBlank(aStyle)) { aStyle = aStyle.replaceAll("background:url\(", ""); aStyle = aStyle.substring(0, aStyle.length() - 29); // aStyle = aStyle.replaceAll("_40x40q90.jpg\) center no-repeat;", ""); System.out.println("aStyle: " + aStyle); } String spanText = liElement.getElementsByTag("span").text(); if (StringUtils.isNotBlank(spanText)) { System.out.println("spanText: " + spanText); } } liHashMap.add(liString); } List<String> combination = test.combination(liHashMap); //获取价格、库存 Elements eles = doc.getElementsByTag("script"); for (Element ele : eles) { String s = ele.toString(); if (!ele.baseUri().contains("tmall")) {//淘宝 String rgex = ""; String subUtilSimple = ""; if (s.contains("skuMap")) { //获取sku的id rgex = "skuMap(.*?)propertyMemoMap"; String skuId = s.replaceAll("\s*", ""); // System.out.println(s); subUtilSimple = getSubUtilSimple(skuId, rgex); subUtilSimple = subUtilSimple.substring(1, subUtilSimple.length() - 1); // JSONObject jb = JSONObject.fromObject(subUtilSimple); JSONObject finalJb = jb; List<String> skuList = new ArrayList<>(); combination.forEach(p->{ JSONObject jsonObject = finalJb.getJSONObject(";" + p + ";"); if (!jsonObject.isNullObject()) { String o = jsonObject.getString("skuId"); System.out.println("sku的id: " + o); skuList.add(o); } }); String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格 String enc = "{"itemNumId":"" + id + ""}"; String substore = ""; String store = ""; try { String gbk = URLEncoder.encode(enc, "utf-8"); String sds = url + gbk; System.out.println("库存、价格" + sds); doc = Jsoup.connect(sds).ignoreContentType(true).get(); store = doc.toString(); rgex = "sku2info(.*?)skuItem"; substore = getSubUtilSimple(store, rgex); substore = substore.substring(3, substore.length() - 3); String sub = substore.replaceAll("\\", "").replaceAll("\s*", ""); JSONObject sb = JSONObject.fromObject(sub); skuList.stream().forEach(p->{ if (sb.has(p)) {//判断是否有值，没值不取，不然会报错 String string = sb.getString(p); System.out.println("淘宝的价格库存==============" + string); } }); } catch (Exception e) { System.out.println("报错的地方store：" + store); // System.out.println("报错的地方substore：" + substore); e.printStackTrace(); System.out.println("=====================================程序报错，提前结束===================================================" ); return; } } if (s.contains("descUrl") && s.contains("counterApi")) { // System.out.println(s); //详情链接 rgex = "protocol(.*?)desc\.alicdn\.com"; subUtilSimple = getSubUtilSimple(s, rgex); subUtilSimple = subUtilSimple.substring(14, subUtilSimple.length() - 7); System.out.println("详情链接: " + subUtilSimple); try { doc = Jsoup.connect("http:" + subUtilSimple).get(); } catch (IOException e) { e.printStackTrace(); } Elements imgDetail = doc.getElementsByTag("img"); for (Element element : imgDetail) { String imgSrc = element.getElementsByTag("img").attr("src"); // imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/",""); // imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据 if (StringUtils.isNotBlank(imgSrc)) { System.out.println("详情图url：" + imgSrc); } } } }else {//天猫的 if (s.contains("TShop.Setup")) { // String rgex = "<bdocid>(.*?)</bdocid>"; String rgex = "skuMap(.*?)salesProp"; String subUtilSimple = getSubUtilSimple(s, rgex); subUtilSimple = subUtilSimple.substring(2, subUtilSimple.length() - 2); JSONObject jb = JSONObject.fromObject(subUtilSimple); List<String> skuList = new ArrayList<>(); combination.forEach(p->{ JSONObject jsonObject = jb.getJSONObject(";" + p + ";"); if (!jsonObject.isNullObject()) { String skuId = jsonObject.getString("skuId"); System.out.println(skuId); skuList.add(skuId); } }); //库存、价格 String url = "https://h5api.m.taobao.com/h5/mtop.taobao.detail.getdetail/6.0/?data=";//手机的html 5 页面 ，为了获取库存、价格 String enc = "{"itemNumId":"" + id + ""}"; String substore = ""; String store = ""; try { String gbk = URLEncoder.encode(enc, "utf-8"); String sds = url + gbk; System.out.println("库存、价格" + sds); doc = Jsoup.connect(sds).ignoreContentType(true).get(); store = doc.toString(); rgex = "sku2info(.*?)skuItem"; substore = getSubUtilSimple(store, rgex); substore = substore.substring(3, substore.length() - 3); String sub = substore.replaceAll("\\", "").replaceAll("\s*", ""); JSONObject sb = JSONObject.fromObject(sub); skuList.stream().forEach(p->{ if (sb.has(p)) {//判断是否有值，没值不取，不然会报错 String string = sb.getString(p); System.out.println("天猫的价格库存==============" + string); } }); } catch (Exception e) { System.out.println("报错的地方store：" + store); // System.out.println("报错的地方substore：" + substore); e.printStackTrace(); System.out.println("=====================================程序报错，提前结束===================================================" ); return; } //详情链接 rgex = "httpsDescUrl(.*?)fetchDcUrl"; subUtilSimple = getSubUtilSimple(s, rgex); subUtilSimple = subUtilSimple.substring(3, subUtilSimple.length() - 3); System.out.println(subUtilSimple); try { doc = Jsoup.connect("http:"+subUtilSimple).get(); } catch (IOException e) { e.printStackTrace(); } Elements imgDetail = doc.getElementsByTag("img"); for (Element element :imgDetail) { String imgSrc = element.getElementsByTag("img").attr("src"); // imgSrc = imgSrc.replaceFirst("//img.alicdn.com/imgextra/",""); // imgSrc = imgSrc.replaceAll("_60x60q90.jpg",""); //处理掉不必要的数据 System.out.println("详情图url："+imgSrc); } break; } } } System.out.println("结束时间：" + new Date()); Date dateEnd = new Date(); long number = dateEnd.getTime()-dateStart.getTime(); //然后在将毫秒转换为date类型就可以了 System.out.println("时间差为： "+number/1000); } /** * 返回单个字符串，若匹配到多个的话就返回第一个，方法与getSubUtil一样 * * @param soap * @param rgex * @return */ public static String getSubUtilSimple(String soap, String rgex) { Pattern pattern = Pattern.compile(rgex);// 匹配的模式 Matcher m = pattern.matcher(soap); while (m.find()) { return m.group(1); } return ""; } /** * 获取指定url中的某个参数 * * @param url * @param name * @return */ public static String getParamByUrl(String url, String name) { url += "&"; String pattern = "(\?|&){1}#{0,1}" + name + "=[a-zA-Z0-9]*(&{1})"; Pattern r = Pattern.compile(pattern); Matcher m = r.matcher(url); if (m.find()) { // System.out.println(m.group(0)); return m.group(0).split("=")[1].replace("&", ""); } else { return null; } } }
import com.google.gson.JsonObject;
import net.sf.json.JSONObject;
import java.util.ArrayList;
import java.util.List;
/**
* Created with Chenquan.
* Description:
* Date: 2018-12-16
* Time: 10:27
*/
public class test {
public static void main(String[] args) {
List<String> li = new ArrayList<>();
li.add("aa");
li.add("bb");
li.add("cc");
List<String> bi = new ArrayList<>();
bi.add("ee");
bi.add("rr");
bi.add("tt");
List<String> ci = new ArrayList<>();
ci.add("yy");
ci.add("uu");
ci.add("ii");
List<List<String>> list = new ArrayList<>();
list.add(li);
list.add(bi);
list.add(ci);
List<String> vv = new ArrayList<>();
List<String> combination = combination(list);
System.out.println(combination);
}
/**
* 若干个集合元素的组合
*
* @param groups 多个集合
* @return 组合结果
*/
public static List<String> combination(List<List<String>> groups) {
if (invalid(groups) || invalid(groups.get(0))) {
return null;
}
List<String> combine = groups.get(0);
for (int i = 1; i < groups.size(); i++) {
combine = cartesianProduct(combine, groups.get(i));
if (combine == null) {
return null;
}
}
return combine;
}
/**
* 两个集合元素的组合
*
* @param c1 集合1
* @param c2 集合2
* @return 组合结果
*/
public static List<String> cartesianProduct(List<String> c1, List<String> c2) {
if (invalid(c1) || invalid(c2)) {
return null;
}
List<String> combine = new ArrayList<>();
for (String s : c1) {
for (String t : c2) {
combine.add(String.format("%s;%s", s, t));
//combine.add(String.format("%s%s", t, s));
}
}
return combine;
}
/**
* 验证集合是否无效
*
* @param c 集合
* @return true 无效
*/
private static boolean invalid(List<?> c) {
return c == null || c.isEmpty();
}
}