HTTP小爬虫 ，nodejs学习(二)

var http = require(‘http‘);
var cheerio = require(‘cheerio‘);
var url = ‘http://www.imooc.com/learn/348‘;

function filter(html){//抓取需要的课程信息
    var $ = cheerio.load(html);
    var chapters = $(‘.chapter‘);
    //var result = [{ chapterTitle:‘‘, videos:[title:‘‘, id:‘‘] }];抓取结果的格式
    var result = [];
    chapters.each(function(){
        var item = $(this);
        var chapterTitle = item.find(‘strong‘).text();
        var videos = item.find(‘.video‘).find(‘li‘);
        var chapterData = {
            chapterTitle: chapterTitle,
            videos: []
        };
        videos.each(function(){
            var video = $(this).find(‘.studyvideo‘);
            var title = video.text().split(‘)‘)[0] + ‘)‘;//去除有用信息后面的空白
            //console.log(title);
            var id = video.attr(‘href‘).split(‘video/‘)[1];//只取课程视频的编号
            chapterData.videos.push({
                title: title,
                id: id
            });
        });
        result.push(chapterData);
    });
    return result;
}

function printResult(result){//打印抓取结果
    var str = ‘‘;
    result.forEach(function(item){
        str += item.chapterTitle + ‘\n‘;
        item.videos.forEach(function(item){
            str += ‘ 【‘ + item.id + ‘】 ‘ + item.title + ‘\n‘;
        });
    });
    console.log(str);
}

http.get(url, function(res){
    var html = ‘‘;
    res.on(‘data‘, function(data){//获取整个页面的所有信息
        html += data;
    });
    res.on(‘end‘, function(){
        var result = filter(html);//过滤该页面，抓取需要的课程信息
        printResult(result);//打印抓取的结果
    });    
}).on(‘error‘, function(){//在获取页面信息时发生错误
    console.log(‘error!‘);
});