import requests
from lxml import html
import time
import pandas as pd
from sqlalchemy import create_engine
import traceback
from fake_useragent import UserAgent as UA #使用其random方法获取随机ua
class ZhaoPinSpider:
def __init__(self):
#http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&sm=0&sg=466cc5c88f044d419cef734bd8713830&p=90
self.url=r"http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC&kw=python&sm=0&sg=466cc5c88f044d419cef734bd8713830&p={}"
self.headers = {'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',
'Accept - Encoding': 'gzip, deflate',
'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',
'Connection': 'Keep-Alive',
'User-Agent': UA().random}
#职位名称
self.Position_Title=[]
# url
self.url_list = []
#反馈率
self.Feedback=[]
#公司名称
self.Company_Name=[]
#职位月薪
self.Monthly_Salary =[]
#工作地点
self.Working_Place =[]
#发布日期
self.Published_Date=[]
self.Position_Title_Key = None
self.Feedback_Key = None
self.Company_Name_Key = None
self.Monthly_Salary_Key = None
self.Working_Place_Key = None
self.Published_Date_Key = None
self.url_key="URL"
self.content_dic={}
self.allinfo_dic = {}
#-------------------------------
self.job_name_list = []
self.corporate_name_list = []
self.benefits_list = []
self.monthly_salary_list = []
self.job_place_list = []
self.published_date_list = []
self.job_nature_list = []
self.job_experience_list = []
self.qualifications_list = []
self.need_worker_num_list = []
self.job_type_list = []
self.job_description_list = []
self.company_introduction_list = []
self.company_scale_list = []
self.company_nature_list = []
self.company_industry_list = []
self.company_url_list = []
self.company_address_list = []
def start(self):
try:
for i in range(1,91):
req = requests.Session()
data = req.get(self.url.format(i), headers=self.headers).text
self.get_info(data,req)
except Exception as e:
print(e)
def get_info(self,source,req):
etree=html.fromstring(source)
# # 职位名称
# self.Position_Title = []
# # 反馈率
# self.Feedback = []
# # 公司名称
# self.Company_Name = []
# # 职位月薪
# self.Monthly_Salary = []
# # 工作地点
# self.Working_Place = []
# # 发布日期
# self.Published_Date = []
TablesNodes=etree.xpath("//div[@class='newlist_list_content']//table")
max=len(TablesNodes)
if TablesNodes:
index=0
for TablesNode in TablesNodes:
if index!=0:
Nodes=TablesNode.xpath(".//tr[1]/td")
TitleNodes=Nodes[0]
url_nodes=TitleNodes.xpath("./div/a[@par]/@href")[0]
# FeedbackNodes=Nodes[1]
# CompanyNodes =Nodes[2]
# SalaryNodes =Nodes[3]
# PlaceNodes =Nodes[4]
# DateNodes = Nodes[5]
# self.Position_Title.append(TitleNodes.text_content().strip())
# FeedbackStr=FeedbackNodes.text_content().strip()
# self.Feedback.append(FeedbackStr if FeedbackStr else "0%")
# self.Company_Name.append(CompanyNodes.text_content().strip())
# self.Monthly_Salary.append(SalaryNodes.text_content().strip())
# self.Working_Place.append( PlaceNodes.text_content().strip())
# self.Published_Date.append( DateNodes.text_content().strip())
if "xiaoyuan" not in url_nodes:
print(url_nodes)
self.get_next_page(url_nodes, req)
self.allinfo_dic.setdefault("job_name", self.job_name_list)
self.allinfo_dic.setdefault("corporate_name", self.corporate_name_list)
self.allinfo_dic.setdefault("benefits", self.benefits_list)
self.allinfo_dic.setdefault("monthly_salary", self.monthly_salary_list)
self.allinfo_dic.setdefault("job_place", self.job_place_list)
self.allinfo_dic.setdefault("published_date", self.published_date_list)
self.allinfo_dic.setdefault("job_nature", self.job_nature_list)
self.allinfo_dic.setdefault("job_experience", self.job_experience_list)
self.allinfo_dic.setdefault("qualifications", self.qualifications_list)
self.allinfo_dic.setdefault("need_worker_num", self.need_worker_num_list)
self.allinfo_dic.setdefault("job_type", self.job_type_list)
self.allinfo_dic.setdefault("job_description", self.job_description_list)
self.allinfo_dic.setdefault("company_introduction", self.company_introduction_list)
self.allinfo_dic.setdefault("company_scale", self.company_scale_list)
self.allinfo_dic.setdefault("company_nature", self.company_nature_list)
self.allinfo_dic.setdefault("company_industry", self.company_industry_list)
self.allinfo_dic.setdefault("company_url", self.company_url_list)
self.allinfo_dic.setdefault("company_address", self.company_address_list)
self.save_Data()
self.allinfo_dic.clear()
self.job_name_list.clear()
self.corporate_name_list.clear()
self.benefits_list.clear()
self.monthly_salary_list.clear()
self.job_place_list.clear()
self.published_date_list.clear()
self.job_nature_list.clear()
self.job_experience_list.clear()
self.qualifications_list.clear()
self.need_worker_num_list.clear()
self.job_type_list.clear()
self.job_description_list.clear()
self.company_introduction_list.clear()
self.company_scale_list.clear()
self.company_nature_list.clear()
self.company_industry_list.clear()
self.company_url_list.clear()
self.company_address_list.clear()
else:
# Nodes = TablesNode.xpath(".//tr[1]/th")
# TitleNodes = Nodes[0]
# FeedbackNodes =Nodes[1]
# CompanyNodes =Nodes[2]
# SalaryNodes = Nodes[3]
# PlaceNodes =Nodes[4]
# DateNodes =Nodes[5]
# self.Position_Title_Key=TitleNodes.text_content().strip()
# self.Feedback_Key=FeedbackNodes.text_content().strip()
# self.Company_Name_Key=CompanyNodes.text_content().strip()
# self.Monthly_Salary_Key=SalaryNodes.text_content().strip()
# self.Working_Place_Key=PlaceNodes.text_content().strip()
# self.Published_Date_Key=DateNodes.text_content().strip()
index = index+1
def save_Data(self):
df=pd.DataFrame(self.allinfo_dic)
df.to_csv("zhaopininfo.csv")
df.to_csv("zhaopininfo.xlsx")
pd.io.sql.to_sql(df,"all_results",schema="zhaopin",con=self.get_engine(),if_exists="append",index=False)
def get_engine(self):
conn=create_engine("mysql+pymysql://root:Aa1234@localhost:3306/zhaopin?charset=utf8mb4")
return conn
def get_next_page(self, nexturl,req):
data=req.get(nexturl).text
self.get_page_info(data)
def get_page_info(self,htmlstr):
try:
root=html.fromstring(htmlstr)
job_name=root.xpath("//div[@class='top-fixed-box' and not(@id)]//div[@class='inner-left fl']/h1/text()")[0]
print(job_name)
corporate_name=root.xpath("//div[@class='top-fixed-box' and not(@id)]//div[@class='inner-left fl']/h2/a/text()")[0]
benefits=root.xpath("//div[@class='top-fixed-box' and not(@id)]//div[@class='inner-left fl']/div[@class='welfare-tab-box']")[0].text_content()
#概要
Nodes = root.xpath("//div[@class='terminalpage-left']//ul/li/strong")
monthly_salary=Nodes[0].text_content().strip()
job_place=Nodes[1].text_content().strip()
published_date=Nodes[2].text_content().strip()
job_nature=Nodes[3].text_content().strip()
job_experience=Nodes[4].text_content().strip()
qualifications=Nodes[5].text_content().strip()
need_worker_num=Nodes[6].text_content().strip()
job_type=Nodes[7].text_content().strip()
job_description=root.xpath("//div[@class='tab-inner-cont']")[0].text_content().strip()
company_introduction=root.xpath("//div[@class='tab-inner-cont']")[1].text_content().strip()
#
companyNodes = root.xpath("//ul[@class='terminal-ul clearfix terminal-company mt20']/li/strong")
company_scale=companyNodes[0].text_content().strip()
company_nature=companyNodes[1].text_content().strip()
company_industry=companyNodes[2].text_content().strip()
company_url=companyNodes[3].text_content().strip()
company_address=None
try:
company_address=companyNodes[4].text_content().strip()
except:
company_address="地址未填写"
self.job_name_list.append(job_name)
self.corporate_name_list.append(corporate_name)
self.benefits_list.append(benefits)
self.monthly_salary_list.append(monthly_salary)
self.job_place_list.append(job_place)
self.published_date_list.append(published_date)
self.job_nature_list.append(job_nature)
self.job_experience_list.append(job_experience)
self.qualifications_list.append(qualifications)
self.need_worker_num_list.append(need_worker_num)
self.job_type_list.append(job_type)
self.job_description_list.append(job_description)
self.company_introduction_list.append(company_introduction)
self.company_scale_list.append(company_scale)
self.company_nature_list.append(company_nature)
self.company_industry_list.append(company_industry)
self.company_url_list.append(company_url)
self.company_address_list.append(company_address)
except:
print(traceback.print_exc())
if __name__=="__main__":
ZhaoPinSpider().start()