案例-爬去南京地区短租房信息

啦啦,网站屏蔽太严重。很难完成

#爬去小猪短租网南京地区短租网13页信息
#导入beautifulsoup库和request库和time库
from bs4 import BeautifulSoup
import requests
import time

#加入请求头:User-Agent，伪装成浏览器
headers={
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}

def judgment_sex(class_name):
    if(class_name==['member_icol']):
        return '女'
    else:
        return '男'
        

def get_links(url):
    web_data=requests.get(url,headers=headers)
    soup=BeautifulSoup(web_data.text,'lxml')
    links=soup.select('#page_list > ul > li > a')
    for link in links:
        href=link.get('href')
        print(href)
        
if __name__=='__main__':
    urls='http://nj.xiaozhu.com/search-duanzufang-p2-0/'
    get_links(urls)

 1 #爬去小猪短租网南京地区短租网13页信息
 2 #导入beautifulsoup库和request库和time库
 3 from bs4 import BeautifulSoup
 4 import requests
 5 import time
 6 
 7 #加入请求头:User-Agent，伪装成浏览器
 8 headers={    
 9     'User-Agent':'Nokia6600/1.0 (3.42.1) SymbianOS/7.0s Series60/2.0 Profile/MIDP-2.0 Configuration/CLDC-1.0'
10     #'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
11 }
12 
13 def judgment_sex(class_name):
14     if(class_name==['member_icol']):
15         return '女'
16     else:
17         return '男'
18         
19 
20 def get_links(url):
21     web_data=requests.get(url,headers=headers)
22     soup=BeautifulSoup(web_data.text,'lxml')
23     links=soup.select('#page_list > ul > li > a')
24     #href=links[0].get('href')
25     #time.sleep(6)
26     #get_info(href)
27     for link in links:
28         href=link.get('href')
29         time.sleep(10)
30         get_info(href)
31     
32     
33 
34 def get_info(url):
35     wb_data=requests.get(url,headers=headers)
36     print(wb_data)
37     print('666666666666666666666666666666')
38     soup=BeautifulSoup(wb_data.text,'lxml')
39     #tittles=soup.select('div.pho_info > h4')
40     #print(tittles)
41     
42     tittles=soup.select(' div.pho_info > h4')
43     addresses=soup.select('span.pr5')    
44     prices=soup.select('#pricePart > div.day_l > span')    
45     imgs=soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > a > img')    
46     names=soup.select('#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a')    
47     sexs=soup.select('#floatRightBox > div.js_box.clearfix > div.member_pic > div')    
48     for tittle,addresse,price,img,name,sex in zip(tittles,addresses,prices,imgs,names,sexs):
49         data={
50             'tittle':tittle.get_text().strip(),
51             'addresse':addresse.get_text().strip(),
52             'price':price.get_text(),
53             'img':img.get("src"),
54             'name':name.get_text(),
55             'sex':judgment_sex(sex.get("class"))
56         }
57         print(data)
58     
59 
60 if __name__=='__main__':
61     urls=['http://nj.xiaozhu.com/search-duanzufang-p{}-0/'.format(number) for number in range(1,15)]
62     for single_url in urls:
63         get_links(single_url)
64         #print(single_url)