Python - 爬取博客园某一目录下的随笔 - 保存为docx

 1 #coding:utf-8
 2 import requests 
 3 from bs4 import BeautifulSoup
 4 import MySQLdb
 5 
 6 
 7 def get_html(url):
 8     ‘‘‘
 9     获取页面HTML源码，并返回
10     ‘‘‘
11     html = requests.get(url)
12     content = html.text.encode(‘utf-8‘)
13     return content
14 
15 def get_blog_html_list(html_content):
16     global addr
17     ‘‘‘
18     在HTML源码中获取到blog相关的值，并返回
19     ‘‘‘
20     bs = BeautifulSoup(html_content,‘lxml‘)
21     # 此处的class要加上下划线，否则会和系统内预定的class冲突
22     blog_list = bs.find_all(‘div‘, class_ = ‘entrylistItem‘)
23     for i in range(len(blog_list)):
24         sub_dict = {}
25         sub_dict[‘title‘] = blog_list[i].a.get_text()
26         sub_dict[‘link‘] = blog_list[i].a.get(‘href‘)
27         sub_dict[‘abstract‘] = blog_list[i].div.get_text()
28         insert_to_db(sub_dict)
29     return sub_dict
30 
31 def insert_to_db(dict={} ):
32     conn = MySQLdb.connect(host = ‘localhost‘, user = ‘root‘, passwd = ‘12345a‘, 
33                            db = ‘pytest01‘, charset = ‘utf8‘)
34     cur = conn.cursor()
35 
36     try:
37         cur.execute(‘insert into blog_details (title, link, abstract) values (%s, %s, %s)‘,
38                      (dict[‘title‘], dict[‘link‘], dict[‘abstract‘]))
39     except MySQLdb.Error, e:
40         pass
41     conn.commit()
42     cur.close()
43     conn.close()

 1 #coding:utf-8
 2 import urllib
 3 import requests
 4 from bs4 import BeautifulSoup
 5 import MySQLdb
 6 import Crawler.get_file
 7 from docx import Document
 8 import time
 9 import docx.image   # 超链接图片抛出的异常在docx.image中
10 
11 index = ‘0‘
12 
13 def get_blog_pic(pic_url, index):
14     print ‘download picture‘
15     ‘‘‘
16     下载指定URL的图片，并且以index命名保存到temp_pic 文件夹内
17     ‘‘‘
18     path = ‘D:\\workspace_Java\\cnBlogCrawler\\temp_pic\\‘ + index + ‘.png‘ 
19     try:
20         pic = urllib.urlretrieve(pic_url, path)
21     except AttributeError,e:
22         path = r‘C:\Users\Lisen\Desktop\error.jpg‘
23         print ‘download picture error, the link is ‘ + pic_url
24         pass
25     finally:
26         return path
27     
28 def get_blog_body(url, title):
29     print ‘download blog‘
30     print title, time.strftime(‘%Y-%m-%d %H:%M:%S‘)
31     file_path = ‘D:\\workspace_Java\\cnBlogCrawler\\temp_doc\\‘ + title + ‘.docx‘
32     header = {‘Accept‘:‘text/html,application/xhtml+xml,application/xml‘,
33               ‘Accept-Encoding‘:‘gzip, deflate, sdch‘,
34               ‘Accept-Language‘:‘zh-CN,zh;q=0.8‘,
35               ‘Cache-Control‘:‘max-age=0‘,
36               ‘Connection‘:‘keep-alive‘,
37               ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36‘}
38     html = requests.get(url,header).text
39     bs = BeautifulSoup(html, ‘lxml‘)
40     print ‘html ready‘
41     ‘‘‘
42     获取html中blog正文部分
43     ‘‘‘
44     body = bs.find_all(‘p‘)
45     ‘‘‘
46     创建文档
47     ‘‘‘    
48     doc = Document()
49     for each in body:
50         # 在函数中改变全局变量需要用global声明下
51         global index
52         ‘‘‘
53         如果p节点有子节点a的话，就说明其中是带着图片的链接或者是文本本身就是链接，因此要分三种情况去爬取
54         1. 没有子节点：直接获取text部分
55         2. 有子节点 a， 但是该子节点没有text部分，说明这是图片
56         3. 有子节点a， 且该子节点有text部分，则里面是文本
57         ‘‘‘
58         if each.a != None:
59             txt = each.a.get_text()
60             if txt == ‘‘:
61                 print ‘pic‘
62                 url = each.a.get(‘href‘)
63                 index = str(int(index) + 1)
64                 pic_name = get_blog_pic(url, index)
65                 try:
66                     doc.add_picture(pic_name)
67                 except docx.image.exceptions.UnrecognizedImageError, e:
68                     pass
69             else:
70                 print ‘URL‘
71                 content = each.a.get_text()
72                 paragraph = doc.add_paragraph(content)
73         else:
74             print ‘txt‘
75             content = each.get_text()
76             paragraph = doc.add_paragraph(content)
77     try:
78         print ‘saving file‘
79         doc.save(file_path)
80     except IOError,e:
81         print ‘file saving error‘
82         print ‘error url is ‘ + url
83         pass

 1 import MySQLdb
 2 import Crawler.get_file
 3 import Crawler.get_blog_details
 4 import time
 5 
 6 if __name__ == ‘__main__‘:
 7 
 8     conn = MySQLdb.connect(host = ‘localhost‘, user = ‘root‘, passwd = ‘12345a‘, 
 9                             db = ‘pytest01‘, charset = ‘utf8‘)
10     cur = conn.cursor()
11     cur.execute(‘truncate blog_details‘)
12     url = r‘http://www.cnblogs.com/omygod/category/460309.html‘
13     print ‘next parsing html‘
14     html = Crawler.get_blog_details.get_html(url)
15     print ‘next parsing blog address‘
16     dict = Crawler.get_blog_details.get_blog_html_list(html)
17     print ‘next update MySQL‘
18     Crawler.get_blog_details.insert_to_db(dict)
19      
20     cur.execute(‘select * from blog_details‘)
21     lines = cur.fetchall()
22     for line in lines:
23         Crawler.get_file.get_blog_body(line[1],line[0])
24         print ‘done‘ + time.strftime(‘%Y-%m-%d %H:%M:%S‘)
25         time.sleep(2)
26         print ‘wait over‘
27     cur.close()
28     conn.close()