C#实现文章采集

结束 结果 采集结果自动保存 采集

完美的一波采集已经结束,想网络上平常所见的文章什么的,相信初学的你懂点什么html就能很熟练的上手了,下面贴一段代码先

 

/*
*BLOGUSER 博主的名字
* http://blog.csdn.net
*/
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using HtmlAgilityPack;

namespace Collector
{

public partial class Form1 : Form
{
// private Task[] tasks = null; //提供线程操作
private const string BLOGUSER = "92jyb"; //博客用户名 
private const string BLOGURL = "http://92jyb.com";
private const string PAGECOUNTPATH = "//div[@id='papelist']/span[1]"; //总页数PATH 
private const string ARTICLEURLPATH = "//span[@class='link_title']/a[1]"; //文章URL的PATH 
private const string ARTICLETITLEPATH = "//div[@class='article_title']/h3/span/a"; //文章标题PATH 
private const string POSTDATEPATH = "//span[@class='link_postdate']"; //文章创建日期PATH 
private const string ARTICLECONTENTPATH = "//div[@id='article_content']"; //文章内容PATH 
private List<string> articleUrlList = new List<string>(); //所有文章的URL 
private object moniter = new object();
private Stopwatch stopwatch = new Stopwatch();
private int cnt = 0;

public Form1()
{
InitializeComponent();
}
/// <summary> 
/// 单篇文章采集 
/// </summary> 
/// <param name="state"></param> 
void CollectArticle(object state) {
Interlocked.Increment(ref cnt);
lock (moniter) {
SetStatuText(string.Format("总共{0}篇文章, 正在采集中第{1}篇.", articleUrlList.Count, cnt));
string articleUrl = (string)state;
string firstArticleHtml = GetHtmlSource(articleUrl, Encoding.UTF8);
if (string.IsNullOrEmpty(firstArticleHtml)) return;
HtmlNode rootNode = GetHtmlNodeByHtml(firstArticleHtml);
string articleTitle = GetNodeInnerText(rootNode, ARTICLETITLEPATH);
string postDate = GetNodeInnerText(rootNode, POSTDATEPATH);
string articleContent = GetNodeInnerText(rootNode, ARTICLECONTENTPATH);
//采集结果处理:保存到数据库或其它...... 
string blogFile = BLOGUSER + ".txt";
using (StreamWriter sw = new StreamWriter(blogFile, true)) {
sw.WriteLine(articleUrl);
sw.WriteLine(articleTitle);
sw.WriteLine(postDate);
sw.WriteLine(articleContent);
}
}
}

private void TaskEnded(Task[] task) {
SetStatuText("采集结束,耗时 " + stopwatch.Elapsed.Minutes + "分" + stopwatch.Elapsed.Seconds + "秒");
} 
//获取总页数 
private int GetPageCount(string pageCountUrl)
{
HtmlNode rootNode = GetHtmlNodeByUrl(pageCountUrl, Encoding.UTF8);
if (rootNode == null)
return 0; 
string pageCountText = GetNodeInnerText(rootNode, PAGECOUNTPATH);
int firstIndex = pageCountText.LastIndexOf("共") + 1;
int lastIndex = pageCountText.LastIndexOf("页");
string result = pageCountText.Substring(firstIndex, lastIndex - firstIndex);
return Convert.ToInt32(result);
}

//通过网页URL获取HtmlNode 
private HtmlNode GetHtmlNodeByUrl(string url, Encoding encoding)
{
string html = GetHtmlSource(url, encoding);
if (string.IsNullOrEmpty(html)) return null;

HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(html);
HtmlNode rootNode = document.DocumentNode;
return rootNode;
}

//通过网页html源代码获取HtmlNode 
private HtmlNode GetHtmlNodeByHtml(string htmlSource)
{
HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
document.LoadHtml(htmlSource);
HtmlNode rootNode = document.DocumentNode;
return rootNode;
}

/// <summary> 
/// 获取网页源代码 
/// </summary> 
private string GetHtmlSource(string url, Encoding encoding)
{
string result = "";
try
{
WebRequest request = WebRequest.Create(url);
using (WebResponse response = request.GetResponse())
using (StreamReader reader = new StreamReader(response.GetResponseStream(), encoding))
result = reader.ReadToEnd();
}
catch
{
result = "";
}
return result;
}

private string GetNodeInnerText(HtmlNode srcNode, string path)
{
HtmlNode temp = srcNode.SelectSingleNode(path);
if (temp == null)
return null;
return temp.InnerText;
}

private void SetStatuText(string s)
{
this.SafeCall(() =>
{
lblStatusInfo.Text = s;

});
}

温馨提示: 此处内容需要评论本文后才能查看.

未经允许不得转载:聚艺帮 » C#实现文章采集

分享到:更多 ()

牛评 抢沙发

爱生活 爱设计

齐聚智慧联系我们