淘宝爬虫

 1 from selenium import webdriver
 2 from selenium.webdriver.common.by import By
 3 from selenium.webdriver.support import expected_conditions as EC
 4 from selenium.webdriver.support.wait import WebDriverWait
 5 from selenium.common.exceptions import TimeoutException
 6 from pyquery import PyQuery as pq
 7 import re
 8 from config import *
 9 import pymongo
10 
11 client = pymongo.MongoClient(MONGO_URL)
12 db = client[MONGO_DB]
13 browser = webdriver.Chrome()
14 
15 wait = WebDriverWait(browser, 10)
16 
17 
18 def search():
19     try:
20         browser.get('https://www.taobao.com')
21         input_ = wait.until(
22             EC.presence_of_element_located((By.CSS_SELECTOR, '#q'))
23         )
24         submit = wait.until(
25             EC.element_to_be_clickable((By.CSS_SELECTOR, '#J_TSearchForm > div.search-button > button'))        
26         )
27 
28         input_.send_keys('xiaomi')
29         submit.click()
30 
31         total = wait.until(
32             EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.total'))
33             )    
34         get_products()
35         return total.text    
36     except TimeoutException:
37         return search()
38 
39 def next_page(page_num):
40     try:
41         input_ = wait.until(
42                 EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > input'))
43             )
44         submit = wait.until(
45                 EC.element_to_be_clickable((By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > div.form > span.btn.J_Submit'))        
46             )
47         input_.clear()
48         input_.send_keys(page_num)
49         submit.click()
50         wait.until(EC.text_to_be_present_in_element(
51             (By.CSS_SELECTOR, '#mainsrp-pager > div > div > div > ul > li.item.active > span'), str(page_num)))
52         get_products()
53     except TimeoutException:
54         next_page(page_num)
55 
56 def get_products():
57     wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, '#mainsrp-itemlist .items .item')))
58     html = browser.page_source
59     doc = pq(html)
60     items = doc('#mainsrp-itemlist .items .item').items()
61     for item in items:
62         product ={
63             'image': item.find('.pic .img').attr('src'),
64             'price': item.find('.price').text(),
65             'deal': item.find('.deal-cnt').text()[:-3],
66             'title': item.find('.title').text(),
67             'shop': item.find('.shop').text(),
68             'location': item.find('.location').text()
69 
70         }
71         print(product)
72         save_to_mongo(product)
73 
74 def save_to_mongo(result):
75     try:
76         if db[MONGO_TABLE].insert(result):
77             print('success save to mongodb', result)
78     except Exception:
79         print('error to mongo')
80 
81 def main():
82     total = search()
83     total = int(re.compile('(d+)').search(total).group(1))
84     # print(total)
85     for i in range(2, total):
86         next_page(i)
87     browser.close()
88 
89 if __name__ == '__main__':
90     main()

config.py

1 MONGO_URL = 'localhost'
2 MONGO_DB = 'taobao'
3 MONGO_TABLE = 'product'

运行结果：

数据库：