多线程队列下载天涯帖子

版本一：


版本二：

  1. #coding:utf-8
  3. import Queue
  4. import threading
  5. import time
  6. import urllib2
  7. import re
  8. import sys
  10. queue = Queue.Queue()
  11. url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
  12. urls=[]
  13. Dict_txt = {}
  14. file_name = 'abc.txt'
  16. class ThreadNum(threading.Thread):
  17.   def __init__(self, queue):
  18.     threading.Thread.__init__(self)
  19.     self.queue = queue
  21.   def run(self):
  22.     while True:
  23.       #消费者端，从队列中获取num
  24.       num = self.queue.get()
  25.       sys.stdout.flush()
  26.       down_text(Dict_txt, num)
  27.       print '**finished download %s**' % num
  28.       time.sleep(1)
  29.       #在完成这项工作之后，使用 queue.task_done() 函数向任务已经完成的队列发送一个信号
  30.       self.queue.task_done()
  33. def down_text(txt_dict, url):
  34.     """根据传入的url抓出各页内容，按页数做键存入字典"""
  35.     print '--downling %s--' % url
  36.     html_content =urllib2.urlopen(url, timeout=30).read()
  37.     text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>s*(.*?)</div>', re.DOTALL)
  38.     text = text_pattern.findall(html_content)
  39.     text_join = ['



'.join(item) for item in text]
  40.     txt_dict[url] = text_join
  42. def page(url):
  43.     """根据第一页地址抓取总页数"""
  44.     html_page = urllib2.urlopen(url, timeout=30).read()
  45.     page_pattern = re.compile(r'<a href="S*?">(d*)</a>s*<a href="S*?" class="S*?">下页</a>')
  46.     page_result = page_pattern.search(html_page)
  47.     list1 = []
  48.     if page_result:
  49.         page_num = int(page_result.group(1))
  50.         page_range = range(1, page_num+1)
  51.         for num in page_range:
  52.                 myurl = '%s%s.shtml' % (url[:-7], num)
  53.                 list1.append(myurl)
  54.     return list1
  56. def write_text(filename, dict, urls):
  57.     """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
  58.     tx_file = open(filename, 'w+')
  59.     for i in urls:
  60.         tx_list = dict[i]
  61.         for tx in tx_list:
  62.             tx = tx.replace('<br>', '
').replace('<br />', '
').replace(' ', '')
  63.             tx_file.write(tx.strip()+'
'*4)
  64.     tx_file.close()   
  67. start = time.time()
  68. def main():
  69.   #产生一个 threads pool, 并把消息传递给thread函数进行处理，这里开启10个并发
  70.   urls = page(url)
  72.   for i in range(5):
  73.     t = ThreadNum(queue)
  74.     t.start()
  75.   
  76.   #往队列中填错数据
  77.   for num in urls:
  78.       queue.put(num)
  79.   #wait on the queue until everything has been processed
  80.   queue.join()
  83.   print '---- start write ----'
  84.   write_text(file_name, Dict_txt, urls)
  86. main()
  87. print"Elapsed Time: %s" % (time.time() - start)


--版本三
  1. #coding:utf-8
  3. import urllib
  4. import re
  5. import threading
  6. import os, time
  8. class Down_Tianya(threading.Thread):
  9.     """多线程下载"""
  10.     def __init__(self, url, num, dt):
  11.         threading.Thread.__init__(self)
  12.         self.url = url
  13.         self.num = num
  14.         self.txt_dict = dt
  16.     def run(self):
  17.         print 'downling from %s' % self.url
  18.         self.down_text()
  20.     def down_text(self):
  21.         """根据传入的url抓出各页内容，按页数做键存入字典"""
  22.         html_content =urllib.urlopen(self.url).read()
  23.         text_pattern = re.compile('<span>时间：(.*?)</span>.*?<!-- <div class="host-ico">楼主</div> -->.*?<div class="bbs-content.*?>s*(.*?)</div>', re.DOTALL)
  24.         text = text_pattern.findall(html_content)
  25.         text_join = ['



'.join(item) for item in text]
  26.         self.txt_dict[self.num] = text_join
  31.   
  32. def page(url):
  33.     """根据第一页地址抓取总页数"""
  34.     html_page = urllib.urlopen(url).read()
  35.     page_pattern = re.compile(r'<a href="S*?">(d*)</a>s*<a href="S*?" class="S*?">下页</a>')
  36.     page_result = page_pattern.search(html_page)
  37.     if page_result:
  38.         page_num = int(page_result.group(1))
  39.         return page_num
  43. def write_text(dict, fn):
  44.     """把字典内容按键（页数）写入文本，每个键值为每页内容的list列表"""
  45.     tx_file = open(fn, 'w+')
  46.     pn = len(dict)
  47.     for i in range(1, pn+1):
  48.         tx_list = dict[i]
  49.         for tx in tx_list:
  50.             tx = tx.replace('<br>', '
').replace('<br />', '
').replace(' ', '')
  51.             tx_file.write(tx.strip()+'
'*4)
  52.     tx_file.close()
  55. def main():
  56.     url = 'http://bbs.tianya.cn/post-16-996521-1.shtml'
  57.     file_name ='abc.txt'
  58.     my_page = page(url)
  59.     my_dict = {}
  61.     print 'page num is : %s' % my_page
  63.     threads = []
  64.    
  65.     """根据页数构造urls进行多线程下载"""
  66.     for num in range(1, my_page+1):
  67.         myurl = '%s%s.shtml' % (url[:-7], num)
  68.         downlist = Down_Tianya(myurl, num, my_dict)
  69.         downlist.start()
  70.         threads.append(downlist)
  71.    
  72.     """检查下载完成后再进行写入"""
  73.     for t in threads:
  74.         t.join()
  76.     write_text(my_dict, file_name)
  78.     print 'All download finished. Save file at directory: %s' % os.getcwd()
  80. if __name__ == '__main__':
  81.     main()





【推广】 免费学中医，健康全家人
原文地址：https://www.cnblogs.com/highroom/p/da69c3af3d0b2be8b4db5946e2de99f1.html