C#/winform采集百度hi文章

public partial class Form1 : Form

{

Thread newth;

public Form1()

{

InitializeComponent();

}

private void buttonGo_Click(object sender, EventArgs e)

{

CheckForIllegalCrossThreadCalls = false; //简单异步线程控制设置

newth = new Thread(new ThreadStart(doit));

newth.Start();

}

void doit()

{ //用HttpWebRequest 对象采集百度hi blog文章

HttpWebRequest webRequest; //请求对象

StreamReader responseReader;//响应对象

string responseData;

html mytml; //自定义html简单处理对象，处理文章页面数据

DataTable dt = new DataTable();//存储文章列表

DataTable dt2 = new DataTable();//存储文章内容

int pagecount = 0;

dt.Columns.Add(new DataColumn("title"));//标题

dt.Columns.Add(new DataColumn("link"));//链接

dt.Columns.Add(new DataColumn("description"));//文章内容

dt.Columns.Add(new DataColumn("pubDate"));//发表时间

dt.Columns.Add(new DataColumn("category"));//文章分类

dt2.Columns.Add(new DataColumn("title"));

dt2.Columns.Add(new DataColumn("link"));

dt2.Columns.Add(new DataColumn("description"));

dt2.Columns.Add(new DataColumn("pubDate"));

dt2.Columns.Add(new DataColumn("category"));

string url = "http://hi.baidu.com/306759613/blog/index/";//文章列表第一页为http://hi.baidu.com/306759613/blog/index/0

string arcurl="http://hi.baidu.com/306759613/blog/item/";//文章所在路径

//find page count

//from index 0

webRequest = WebRequest.Create(url + 0) as HttpWebRequest;

webRequest.Timeout = 3000;//请求延时设置

WebResponse reponse = webRequest.GetResponse();

//以gb2312读取数据

responseReader = new StreamReader(

reponse.GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312")

);

responseData = responseReader.ReadToEnd();//读取整个页面

responseReader.Close();

mytml = new html(responseData);//创建html页面处理对象

List<string> regpsl = mytml.getElementsByRegex(@"/blog/index/[\d]+");//获取分页链接的正则

List<int> pagenum = new List<int>();//存储页码

foreach (string a in regpsl) {

pagenum.Add(int.Parse(a.Replace("/blog/index/", "")));

}

pagecount = pagenum.Max() + 1;//pagenum中最大值为尾页页码，页面从0开始编号，页数为页面数+1

mytml = null;

this.progressBar1.Value = 0; //进度条

for (int i = 0; i < pagecount; i++)

{

webRequest = WebRequest.Create(url+i) as HttpWebRequest;//读取各分页

webRequest.Timeout = 3000;

responseReader = new StreamReader(

webRequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312")

);

responseData = responseReader.ReadToEnd();

responseReader.Close();

mytml = new html(responseData);

DataTable dti = mytml.getAritcleTable();//获取该分页文章列表

this.progressBar1.Value = (i * 100 / pagecount);

this.label1.Text = this.progressBar1.Value + "%"; //进度条

for (int j = 0; j < dti.Rows.Count; j++)

{

dt.Rows.Add(dti.Rows[j].ItemArray);//插入该文章到总文章表

HttpWebRequest subrequest = WebRequest.Create(arcurl+dti.Rows[j][1]+".html") as HttpWebRequest;//读取文章信息

subrequest.Timeout = 3000;

StreamReader subre = new StreamReader (subrequest.GetResponse().GetResponseStream(), System.Text.Encoding.GetEncoding("gb2312"));

string tmphtml = subre.ReadToEnd();

//处理文章页面html开始

int start = tmphtml.IndexOf("<div id=\"blog_text\" class=\"cnt\">")+"<div id=\"blog_text\" class=\"cnt\">".Length;

int end = tmphtml.IndexOf("</div",start);

tmphtml = tmphtml.Substring(start,end-start);//取得文章内容

dt2.Rows.Add(new object[] { dti.Rows[j].ItemArray[0], dti.Rows[j].ItemArray[1], tmphtml, dti.Rows[j].ItemArray[3] });//插入文章数据到文章表

subre.Close();

this.progressBar1.Value = (i * 100 / pagecount) + (j * 25 / dti.Rows.Count);//设置进度条

this.label1.Text = this.progressBar1.Value + "%"; //显示百分比

writeXML(dt2, "f:\\p\\" + dti.Rows[j][1] + ".xml");//将文章以xml格式输出

dt2.Rows.Clear();

subre.Close();

subrequest = null;

}

webRequest = null;

responseReader.Close();

responseReader = null;

responseData = string.Empty;

}

this.progressBar1.Value =this.progressBar1.Maximum;//进度100%

this.label1.Text = this.progressBar1.Value + "%";

this.dataGridView1.DataSource = dt;//显示文章列表数据

writeXML(dt, "f:\\p\\Articel.xml");//输出文章内容数据到xml文件

textBoxDebug.Text = textBoxDebug.Text+ "写入完毕\r\n";

}

/// <summary>

/// 将数据表输出到xml

/// </summary>

/// <param name="dt"></param>

/// <param name="fileName"></param>

public void writeXML(DataTable dt, string fileName)

{

string xmlstr ="<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n";

xmlstr += "<?xml-stylesheet href=\"t.xsl\" type=\"text/xsl\"?>\r\n";

xmlstr +="<root>\r\n";

dt.TableName = "articels";

System.Xml.XmlDocument xml = new System.Xml.XmlDocument();

for (int k = 0; k < dt.Rows.Count; k++)

{

xmlstr = xmlstr + "<" + dt.TableName + ">\r\n";

for (int l = 0; l < dt.Columns.Count; l++)

{

xmlstr = xmlstr + "<" + dt.Columns[l].ColumnName + ">\r\n<![CDATA[\r\n";

xmlstr = xmlstr + dt.Rows[k][l] + "\r\n";

xmlstr = xmlstr + "]]>\r\n</" + dt.Columns[l].ColumnName + ">\r\n";

}

xmlstr = xmlstr + "</" + dt.TableName + ">\r\n";

}

xmlstr += "</root>\r\n";

StreamWriter w = new StreamWriter(fileName, false, System.Text.Encoding.UTF8);//以utf8保存

w.Write(xmlstr);

w.Close();

}

class html

{

string htmltext=string.Empty;

/// <summary>

/// 构造函数

/// </summary>

/// <param name="htmltext"></param>

public html( string htmltext) {

this.htmltext = htmltext;

}

/// <summary>

/// 获取文章列表

/// </summary>

/// <returns></returns>

public DataTable getAritcleTable(){

DataTable dt = new DataTable();

int start = htmltext.IndexOf("div id=\"m_blog\" class=\"modbox\">");//起始位置

int end = htmltext.IndexOf("<div id=\"mod_artclg\" class=\"mod\">");//结束位置

string htm = htmltext.Substring(start-1, end - start -1 );

dt.Columns.Add(new DataColumn("title"));

dt.Columns.Add(new DataColumn("link"));

dt.Columns.Add(new DataColumn("description"));

dt.Columns.Add(new DataColumn("pubDate"));

dt.Columns.Add(new DataColumn("category"));

string title, link, description, pubDate, category,temp;

int nstart, nend;//记录上次提取位置

start = 0;

{//遍历html文档提取文章信息

nstart = htm.IndexOf("<div class=\"tit\">",start) + "<div class=\"tit\">".Length;

if (nstart < start) break;

start = nstart;

nend = htm.IndexOf("</div>",start);

start = nend + 5;

temp = htm.Substring(nstart, nend - nstart );

nstart = temp.IndexOf(">");

nend =temp.IndexOf("</a>");

title = temp.Substring(nstart + 1, nend-nstart-1 );//文章标题

nstart = temp.IndexOf("\"");

nend = temp.IndexOf("\"", nstart + 1);

link = temp.Substring(nstart + 1, nend - nstart-1 );//链接

nstart = link.IndexOf("item/")+"item/".Length;

nend = link.IndexOf(".html");

link = link.Substring(nstart, nend - nstart);//取文件名（去除扩展名）

nstart = htm.IndexOf("<div class=\"date\">", start)+ "<div class=\"date\">".Length;

start = nstart;

nend = htm.IndexOf("</div>", start);

pubDate = htm.Substring(nstart , nend - nstart);//发表日期

start = nend + 5;

nstart = htm.IndexOf("<div class=\"cnt\">", start) + "<div class=\"cnt\">".Length;

start = nstart;

nend = htm.IndexOf("</div>", start);

start = nend + 5;

description = htm.Substring(nstart, nend - nstart );//文章内容

nstart = htm.IndexOf("<div class=\"opt\">", start) + "<div class=\"opt\">".Length;

start = nstart;

nend = htm.IndexOf("</div>", start);

start = nend + 5;

temp = htm.Substring(nstart, nend - nstart );

nstart = temp.IndexOf("：");

nend =temp.IndexOf("</a>");

category=temp.Substring(nstart + 1, nend - nstart - 1); //文章分类

dt.Rows.Add(new string[] { title, link, description, pubDate, category });

} while (nstart > 0);

return dt;

}