三种数据解析方式

from lxml import etree

#xpath 函数返回的总是一个列表

#创建etree对象进行制定数据解析
tree = etree.parse(‘./test.html‘)
#属性定位：根据指定属性定位到指定的节点标签
tree.xpath(‘//div[@class="song"]‘)
# 层级&索引定位：找到class属性为tang的div的直系子标签ul下的第二个子标签li下的直系子标签a。 【是从1开始的】
tree.xpath(‘//div[@class="tang"]/ul/li[2]/a‘)
# 逻辑定位：
tree.xpath(‘//a[@href="" and @class="du"]‘)
# 模糊匹配：
tree.xpath(‘//div[contains(@class,"ng")]‘)    #找到class属性里包含ng的class
tree.xpath(‘//div[start-with(@class,"ta")]‘) #找到class属性里以ta开头的class
# 取文本
tree.xpath(‘//div[@class="song"]/p[1]/text()‘) #/text()获取当前标签下的直系储存的文本数据
tree.xpath(‘//div[@class="tang"]//text()‘)    # # //text()获取某一个标签下所有子标签中存储的文本数据

# 取属性
tree.xpath(‘//div[@class="tang"]//li[2]/a/@href‘)

import requests
from lxml import etree

#1. 指定url
url = ‘https://ishuo.cn/joke‘
#2. 发送请求
headers = {
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36‘
}
response = requests.get(url=url,headers=headers)
#3. 获取页面内容
page_text = response.text
#4. 数据解析
tree = etree.HTML(page_text)
#获取所有li标签（段子和标题都在li标签中）
li_list=tree.xpath(‘//div[@id="list"]/ul/li‘)
####注意：Element类型的对象可以继续调用xpath函数，对该对象表示的局部内容进行指定内容的解析 ########
f = open(‘./duanzi.txt‘,‘w‘,encoding=‘utf8‘)
for li in li_list:   #每个li对象都是Element类型
    content = li.xpath(‘./div[@class="content"]/text()‘)[0]
    title = li.xpath(‘./div[@class="info"]/a/text()‘)[0]
#5. 持久化存储
    f.write(title+":"+content+"\n\n")
    print(‘一条数据写入成功‘)

rom bs4 import BeautifulSoup

fp=open(‘text.html‘)
soup = BeautifulSoup(fp,‘lxml‘)

import requests
from bs4 import BeautifulSoup

url = ‘http://www.shicimingju.com/book/sanguoyanyi.html‘
headers = {
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36‘
}
def get_content(url):
    content_page = requests.get(url=url,headers=headers).text
    soup = BeautifulSoup(content_page,‘lxml‘)
    content=soup.find(‘div‘,class_=‘chapter_content‘).text
    return content
page_text = requests.get(url=url,headers=headers).text
#数据解析
soup = BeautifulSoup(page_text,‘lxml‘)
#a_list列表中存储的是一系列的a标签对象
a_list = soup.select(‘.book-mulu > ul > li > a ‘)
# type(a_list[0]) 返回bs4.element.Tag
#注意：Tag类型的对象可以继续调用相应的解析属性和方法进行局部数据的解析
f = open(‘sanguo.txt‘,‘w‘,encoding=‘utf8‘)
for a in a_list:
    title = a.string
    content_url = ‘http://www.shicimingju.com‘ + a[‘href‘]
    content = get_content(content_url)
    f.write(title+":"+content+"/n/n/n")
    print(‘写入一个章节‘)
    

import requests
import re,os
#指定url

url = ‘https://www.qiushibaike.com/pic/‘

headers = {
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36‘
}

page_text = requests.get(url=url,headers=headers).text

#数据解析(img_list中储存的就是当前页面源码中所有的图片链接)

# img_list = re.findall(‘<div class="thumb">.*?<img src="(.*?)".*?>.*?</div>‘,page_text,re.S)
img_list = re.findall(‘<div class="thumb">.*?<img src="(.*?)".*?</div>‘,page_text,re.S)

#创建一个存储数据的文件夹

if not os.path.exists(‘./qiubai_img‘):
    os.mkdir(‘qiubai_img‘)

for url in img_list:
    #将图片的url进行拼接，拼成完整的url
    img_url = ‘https:‘+ url
    #持久化存储：存储的是图片数据，不是url
    #获取图片的二进制数据
    img_data = requests.get(url=img_url,headers=headers).content
#     print(img_data)
    imgName = img_url.split(‘/‘)[-1]
#     print(imgName)
    imgPath = ‘qiubai_img/‘+ imgName
    print(imgPath)
    with open(imgPath,‘wb‘) as f:
        f.write(img_data)
        print(imgName+‘写入成功‘)