nodejs .http模块, cheerio模块 实现 小爬虫.

代码:

var http = require("http");

var cheerio = require("cheerio");


var url = 'http://www.imooc.com/learn/348';


http.get(url, function(res){
    var html = '';

    res.on('data', function(data){
        html += data;
    });

    res.on('end', function(){
        var courseData = filterChapters(html);

        printCourseInfo(courseData);
        console.log(courseData);
    });
}).on('error', function(){
    console.log("获取课程数据出错!");
});

function filterChapters(html)
{
    var $ = cheerio.load(html);

    //所有章节
    var chapters = $('.chapter');

    var courseData = [];

    chapters.each(function(item){
        var chapter = $(this);
        var chapterTitle = chapter.find('h3 strong').text().replace(/
|
/ig,"").trim();
        var videos = chapter.find(".video").children('li');

        var chapterData = {
            chapterTitle: chapterTitle,
            videos: []
        };

        videos.each(function(index, item2) {
            var video = $(this).find('.J-media-item');
            var videoTitle = video.text().replace(/
|
/ig,"").trim();
            var id = video.attr('href').split('video/')[1];

            chapterData.videos.push({
                title: videoTitle,
                id: id
            })
        });

        courseData.push(chapterData);
    });

    return courseData;
}


function printCourseInfo(courseData)
{
    courseData.forEach(function(item){
        var chapterTitle = item.chapterTitle;
        console.log(chapterTitle + '
');

        item.videos.forEach(function(video){
            console.log('   [' + video.id+ ']' + video.title);
        });
    });
}

运行:

----------------------------------------------------------------------

参考链接:

• http小爬虫
• Node.js的学习--使用cheerio抓取网页数据