标签:
1 #coding:utf-8 2 import requests 3 from bs4 import BeautifulSoup 4 import MySQLdb 5 6 7 def get_html(url): 8 ‘‘‘ 9 获取页面HTML源码,并返回 10 ‘‘‘ 11 html = requests.get(url) 12 content = html.text.encode(‘utf-8‘) 13 return content 14 15 def get_blog_html_list(html_content): 16 global addr 17 ‘‘‘ 18 在HTML源码中获取到blog相关的值,并返回 19 ‘‘‘ 20 bs = BeautifulSoup(html_content,‘lxml‘) 21 # 此处的class要加上下划线,否则会和系统内预定的class冲突 22 blog_list = bs.find_all(‘div‘, class_ = ‘entrylistItem‘) 23 for i in range(len(blog_list)): 24 sub_dict = {} 25 sub_dict[‘title‘] = blog_list[i].a.get_text() 26 sub_dict[‘link‘] = blog_list[i].a.get(‘href‘) 27 sub_dict[‘abstract‘] = blog_list[i].div.get_text() 28 insert_to_db(sub_dict) 29 return sub_dict 30 31 def insert_to_db(dict={} ): 32 conn = MySQLdb.connect(host = ‘localhost‘, user = ‘root‘, passwd = ‘12345a‘, 33 db = ‘pytest01‘, charset = ‘utf8‘) 34 cur = conn.cursor() 35 36 try: 37 cur.execute(‘insert into blog_details (title, link, abstract) values (%s, %s, %s)‘, 38 (dict[‘title‘], dict[‘link‘], dict[‘abstract‘])) 39 except MySQLdb.Error, e: 40 pass 41 conn.commit() 42 cur.close() 43 conn.close()
1 #coding:utf-8 2 import urllib 3 import requests 4 from bs4 import BeautifulSoup 5 import MySQLdb 6 import Crawler.get_file 7 from docx import Document 8 import time 9 import docx.image # 超链接图片抛出的异常在docx.image中 10 11 index = ‘0‘ 12 13 def get_blog_pic(pic_url, index): 14 print ‘download picture‘ 15 ‘‘‘ 16 下载指定URL的图片,并且以index命名保存到temp_pic 文件夹内 17 ‘‘‘ 18 path = ‘D:\\workspace_Java\\cnBlogCrawler\\temp_pic\\‘ + index + ‘.png‘ 19 try: 20 pic = urllib.urlretrieve(pic_url, path) 21 except AttributeError,e: 22 path = r‘C:\Users\Lisen\Desktop\error.jpg‘ 23 print ‘download picture error, the link is ‘ + pic_url 24 pass 25 finally: 26 return path 27 28 def get_blog_body(url, title): 29 print ‘download blog‘ 30 print title, time.strftime(‘%Y-%m-%d %H:%M:%S‘) 31 file_path = ‘D:\\workspace_Java\\cnBlogCrawler\\temp_doc\\‘ + title + ‘.docx‘ 32 header = {‘Accept‘:‘text/html,application/xhtml+xml,application/xml‘, 33 ‘Accept-Encoding‘:‘gzip, deflate, sdch‘, 34 ‘Accept-Language‘:‘zh-CN,zh;q=0.8‘, 35 ‘Cache-Control‘:‘max-age=0‘, 36 ‘Connection‘:‘keep-alive‘, 37 ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36‘} 38 html = requests.get(url,header).text 39 bs = BeautifulSoup(html, ‘lxml‘) 40 print ‘html ready‘ 41 ‘‘‘ 42 获取html中blog正文部分 43 ‘‘‘ 44 body = bs.find_all(‘p‘) 45 ‘‘‘ 46 创建文档 47 ‘‘‘ 48 doc = Document() 49 for each in body: 50 # 在函数中改变全局变量需要用global声明下 51 global index 52 ‘‘‘ 53 如果p节点有子节点a的话,就说明其中是带着图片的链接或者是文本本身就是链接,因此要分三种情况去爬取 54 1. 没有子节点:直接获取text部分 55 2. 有子节点 a, 但是该子节点没有text部分,说明这是图片 56 3. 有子节点a, 且该子节点有text部分,则里面是文本 57 ‘‘‘ 58 if each.a != None: 59 txt = each.a.get_text() 60 if txt == ‘‘: 61 print ‘pic‘ 62 url = each.a.get(‘href‘) 63 index = str(int(index) + 1) 64 pic_name = get_blog_pic(url, index) 65 try: 66 doc.add_picture(pic_name) 67 except docx.image.exceptions.UnrecognizedImageError, e: 68 pass 69 else: 70 print ‘URL‘ 71 content = each.a.get_text() 72 paragraph = doc.add_paragraph(content) 73 else: 74 print ‘txt‘ 75 content = each.get_text() 76 paragraph = doc.add_paragraph(content) 77 try: 78 print ‘saving file‘ 79 doc.save(file_path) 80 except IOError,e: 81 print ‘file saving error‘ 82 print ‘error url is ‘ + url 83 pass
1 import MySQLdb 2 import Crawler.get_file 3 import Crawler.get_blog_details 4 import time 5 6 if __name__ == ‘__main__‘: 7 8 conn = MySQLdb.connect(host = ‘localhost‘, user = ‘root‘, passwd = ‘12345a‘, 9 db = ‘pytest01‘, charset = ‘utf8‘) 10 cur = conn.cursor() 11 cur.execute(‘truncate blog_details‘) 12 url = r‘http://www.cnblogs.com/omygod/category/460309.html‘ 13 print ‘next parsing html‘ 14 html = Crawler.get_blog_details.get_html(url) 15 print ‘next parsing blog address‘ 16 dict = Crawler.get_blog_details.get_blog_html_list(html) 17 print ‘next update MySQL‘ 18 Crawler.get_blog_details.insert_to_db(dict) 19 20 cur.execute(‘select * from blog_details‘) 21 lines = cur.fetchall() 22 for line in lines: 23 Crawler.get_file.get_blog_body(line[1],line[0]) 24 print ‘done‘ + time.strftime(‘%Y-%m-%d %H:%M:%S‘) 25 time.sleep(2) 26 print ‘wait over‘ 27 cur.close() 28 conn.close()
由于博客园的原创博客都是通过随笔的形式保存的,因此我们可以通过对某一随笔目录进行解析,获取出该目录下所有博文的标题,链接以及摘要,存储到MySQL数据库中(主要是因为可以持久记录相关信息,后续有新博文的时候可以通过对比判断直接下载新的博文)。然后再对每个条目进行单独解析,将博文的内容,图片保存到Word文档中。
主要用到的包有: requests, BeautifulSoup,python-docx, MySQLdb
整个代码分为三部分:
在编写代码过程中,有几个地方需要注意:
现阶段缺点:
体会:
1. 真的真的要做异常处理,否则会很痛苦
2. 分析不全面,没有考虑到含有表格或者其他形式
3. 争取做出GUI,方便处理
4. 添加log 文档, 方便查看出错的地方
Python - 爬取博客园某一目录下的随笔 - 保存为docx
标签:
原文地址:http://www.cnblogs.com/sap-bw-lcz/p/5467513.html