python 爬取qidian某一页全部小说

  1 import re
  2 import urllib.request
  3 from bs4 import BeautifulSoup
  4 import time
  5 
  6 url=input("第一页网址：")
  7 
  8 def gethtml(url):
  9                                       #获取页面源代码html
 10     page=urllib.request.urlopen(url)
 11     html=page.read().decode(‘utf-8‘)  #html是一个列表
 12     soup=BeautifulSoup(html,‘html.parser‘)
 13     
 14     return soup
 15 
 16 
 17 def getbookurl(soup):                   #获取该页所有书本的链接地址
 18     firsturl2=[]
 19     bookurl=soup.find_all("h4")
 20     bookurl1=re.findall(r‘<h4><a data-bid=".*?" data-eid=".*?" href="(.*?)" target="_blank"‘,str(bookurl))
 21     for i in range(0,len(bookurl1)):
 22         bookurl="http:"+bookurl1[i]
 23        
 24         soup1=gethtml(bookurl)          #获取每本书第一章 的url
 25         time.sleep(0.2)
 26         firsturl=soup1.find_all("a",{"class":"red-btn J-getJumpUrl "})
 27         firsturl1=re.findall(r‘data-firstchapterjumpurl=".*?" href="(.*?)" id="readBtn">‘,str(firsturl))
 28         if firsturl1[0]==‘‘:            #这里要进行判断，防止出错
 29             continue
 30         firsturl2.append(firsturl1[0])
 31     return firsturl2
 32 
 33 
 34 
 35 
 36 def getcontent(soup,load):
 37     
 38     content=soup.find_all("div",{"class":"read-content j_readContent"})
 39     
 40     content1=re.compile(r‘<p>([\s\S]*?)</p>‘)
 41     
 42     content2=content1.findall(str(content))
 43    
 44     content3=re.sub("</?\w+[^>]*>",‘‘,content2[0])
 45     
 46     content4=content3.replace(‘。‘,‘。\n\n\0\0\0‘)  #到此，将章节内容获取完毕
 47 
 48     contentname=re.compile(r‘<h3 class="j_chapterName">(.*?)</h3>‘)
 49     
 50     contentname1=contentname.findall(str(soup))     #获取章节名称
 51 
 52     book="----------------------------------------------------------------"+contentname1[0]+"------------------------------------------------------------\n\n\n"+content4   
 53 
 54     with open(load, ‘a‘) as f:
 55 
 56         f.write(book)
 57 
 58     
 59 
 60 def nextcontent(soup):
 61 
 62     content=soup.find_all("div",{"class":"chapter-control dib-wrap"})
 63     
 64     #print(str(content))
 65     
 66     step=re.compile(r‘<a data-eid="qd_R109" href="(.*?)" id="j_chapterNext">‘)
 67 
 68     content1=step.findall(str(content))
 69 
 70     if content1 == []:
 71 
 72         step1=re.compile(r‘<a data-eid="qd_R118" href="(.*?)" id="j_chapterNext">‘)
 73 
 74         content2=step1.findall(str(content))
 75 
 76         url="http:"+content2[0]
 77 
 78         return url
 79     else:
 80         url="http:"+content1[0]
 81 
 82         return url
 83 
 84 def panduan(soup):
 85     
 86     content=soup.find_all("div",{"class":"chapter-control dib-wrap"})
 87     
 88     #print(str(content))
 89     
 90     step=re.compile(r‘<a data-eid="qd_R109" href="(.*?)" id="j_chapterNext">‘)
 91     
 92     content1=step.findall(str(content))
 93     
 94     return content1
 95     #-------------------------------------------------------------------------
 96     
 97     
 98     
 99     #-------------------------------------------------------------------------
100     
101 while 1==1:
102     soup2=gethtml(url)
103     firsturl2=getbookurl(soup2)
104 
105     for j in range(0,len(firsturl2)):
106         url="http:"+firsturl2[j]
107         soup1=gethtml("http:"+firsturl2[j])
108         bookname=re.findall(r‘<h1>(.*?)</h1>‘ ,str(soup1))
109         load="d:/88/%s.txt" % bookname[0]
110         i=0
111         while 1==1:
112             soup=gethtml(url)
113             getcontent(soup,load)
114             url=nextcontent(soup)
115             content1=panduan(soup)
116             i+=1
117             print("第%d章下载完成" % i)
118     
119             if content1 == []:
120                 break
121             
122             time.sleep(0.2)
123         print("-------------第%d本书下载完成---------" % int(j+1))
124