前一段时间用Python写了个简单的网络爬虫，可以从某个求职网站上根据预先设置的条件一次性的爬取所有的职位信息，最近对该程序进行了一下完善，主要包括如下内容
(1)可以对爬取的结果再进行筛选
例如，你开始预设的条件是“java高级软件工程师”，会列出很多java相关的职位，但是如果你只想看看ibm,hp,huawei这几家公司的职位信息，很多职位网站是不支持的。目前这只小虫子可以通过 FAV_COMPANY列表来赛选
(2)增加了职位的详细信息页面的URL
目前 列出了 职位名称，公司名称，详细信息URL
(3)增加了Email的功能
可以预先设置相关的邮箱配置，然后将搜索到的结果发送到邮箱，如果你有兴趣，可以再通过timer的方式，定期的对某个网站进行职位爬取，如果有合适的就发送到自己的邮箱，这样就不用天天登录网站进行查询了
下面是相关的代码，希望可以给你带来帮助，如果什么不足，欢迎反馈
'''
(1)FAV_COMPANY:列出了有兴趣的公司名称,如果为空将会列出所有的公司
(2)可以将结果发送到制定的邮箱
'''
import urllib.request
import re
import smtplib
import time
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
#计数器
PAGE_NUMBER = 1
TOTAL_JOB_COUNT = 0;
#搜索内容以及打印的配置
FAV_COMPANY = []
#FAV_COMPANY = ['IBM', 'HP', '华为', '腾讯']
JOB_NAME_TITLE = "职位名称:"
COM_NAME_TITLE = "公司名称:"
DETAIL_TITLE = "明细页面:"
NEED_SEND_EMAIL=True
#邮件的相关配置
FROM_EMAIL_ADDRESS = "test@126.com"
TO_EMAIL_ADDRESS = "test@gmail.com"
CC_EMAIL_ADDRESS = "test@gmail.com"
ATT_FILE_NAME="job_infor.txt"
SMTP_SERVER = "smtp.126.com"
SMTP_USER_NAME = 'test'
SMTP_USER_PASSWORD = 'test'
def filter_job(url):
#Python 3.X 必须用 urllib.request打开一个URL
text = urllib.request.urlopen(url).read().decode("gbk")
page_navi(text)
#页面跳转
def page_navi(page_source):
#第一页直接获取信息
extract_job_info(page_source)
#"下一页"链接的正则
next_page_regular = r'</td><td><a href=(.*?)class="orange1".+?style=.*?>(.*?)<img .*?pageron.gif.*? />.*?</a></td></tr>'
next_pagelink_set = re.findall(next_page_regular, page_source)
if len(next_pagelink_set) > 0:
#如果需要更改全局变量,则需要先用global来声明
global PAGE_NUMBER
PAGE_NUMBER = PAGE_NUMBER + 1
print("#"*10 , "Page Number:", str(PAGE_NUMBER) , "#"*10, "Record Number:", TOTAL_JOB_COUNT)
next_url = next_pagelink_set[0][0].split('"')[1]
#解析出"下一页"按钮对应的链接,然后递归调用
filter_job(next_url)
#内容抽取
def extract_job_info(page_source):
#职位名称的正则
job_name_reqular = r'<a .*? class="jobname" .*>(.*?)</a>'
#公司名称以及明细页面的正则
com_name_reqular = r'<a href=(.*?) class="coname" target="_blank".*>(.*?)</a>'
job_name_set = re.findall(job_name_reqular, page_source)
com_name_set = re.findall(com_name_reqular, page_source)
global TOTAL_JOB_COUNT
TOTAL_JOB_COUNT = TOTAL_JOB_COUNT + len(com_name_set)
#使用zip()对结果进行展示
job_information=open(ATT_FILE_NAME,'w+')
for job_name, com_name in zip(job_name_set, com_name_set):
if len(FAV_COMPANY) > 0:
for expected_com_name in FAV_COMPANY:
if expected_com_name.lower() in com_name[1].lower():
#send_email(job_name, com_name)
deal_with_result(job_name, com_name,job_information)
else:
deal_with_result(job_name, com_name,job_information)
job_information.close()
def email():
if NEED_SEND_EMAIL:
time.asctime(time.localtime(time.time()))
subject=time.asctime(time.localtime(time.time()))+" job information"
content=subject
if len(FAV_COMPANY) >0:
content=content+"
fav company list is "+ str(FAV_COMPANY)
send_email(subject,content)
#对进过处理
def deal_with_result(job_name, com_name,job_information):
print(JOB_NAME_TITLE, job_name)
print(COM_NAME_TITLE, com_name[1])
print(DETAIL_TITLE, com_name[0], "
")
if NEED_SEND_EMAIL:
job_information.writelines(JOB_NAME_TITLE+job_name)
job_information.writelines(COM_NAME_TITLE+com_name[1])
job_information.writelines(DETAIL_TITLE+com_name[0]+"
")
def send_email(subject, content):
msg = MIMEMultipart()
msg['From'] = FROM_EMAIL_ADDRESS
msg['To'] = TO_EMAIL_ADDRESS
msg['Cc'] = CC_EMAIL_ADDRESS
msg['Subject'] = subject
# add content
txt = MIMEText(content, 'html', 'utf-8')
msg.attach(txt)
# add attach file
fileName = ATT_FILE_NAME
att = MIMEText(open(fileName, 'rb').read(), 'base64', 'gb2312')
att["Content-Type"] = 'application/octet-stream'
att["Content-Disposition"] = 'attachment; filename="attch.txt"'
msg.attach(att)
# send email
smtp = smtplib.SMTP(SMTP_SERVER)
smtp.set_debuglevel(1)
smtp.login(SMTP_USER_NAME, SMTP_USER_PASSWORD)
smtp.sendmail(FROM_EMAIL_ADDRESS, TO_EMAIL_ADDRESS, msg.as_string())
smtp.quit()
print("send successful")
#发送邮件的测试方法
def test_send_email():
for i in range(100):
subject = "Python test email:" + str(i);
content = "<br/><font size="2" face="Verdana"><b>" + "This is contents" + str(i) + "</b></font><br/>"
send_email(subject, content)
#正则表达式的测试方法
def test_regular():
url_reqular = r'<a.*?class="coname".*?target="_blank".*?href=(.*?)>(.*?)</a>'
url = "<a class="coname" target="_blank" href=http://search.job.com/list/co,c,2766722,000000,10,1.html>上海锐格软件有限公司</a>dddd"
print("URL:", re.findall(url_reqular, url))
#entrance function
if __name__ == "__main__":
#输入一定条件后,结构列表的首页URL,只要输入这个作为条件
url='''http://search.job.com/jobsearch/search_result.php?fromJs=1&jobarea=180200%2C00&funtype=0000&industrytype=00&keyword=python&keywordtype=2&lang=c&stype=1&postchannel=0000&fromType=23'''
filter_job(url)
email()