爬虫基础-2

# coding=utf-8
import requests
import json
import sys

class BaiduFanyi:
    def __init__(self,trans_str):
        self.trans_str = trans_str
        self.lang_detect_url = "http://fanyi.baidu.com/langdetect"
        self.trans_url = "http://fanyi.baidu.com/basetrans"
        self.headers = {"User-Agent":"Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36"}

    def parse_url(self,url,data): #发送post请求，获取响应
        response = requests.post(url,data=data,headers=self.headers)
        return json.loads(response.content.decode())

    def get_ret(self,dict_response):#提取翻译的结果
        ret = dict_response["trans"][0]["dst"]
        print("result is :",ret)


    def run(self):#实现主要逻辑
        #1.获取语言类型
            #1.1 准备post的url地址，post_data
        lang_detect_data = {"query":self.trans_str}
            #1.2 发送post请求，获取响应
        lang = self.parse_url(self.lang_detect_url,lang_detect_data)["lan"]
            #1.3 提取语言类型
        #2.准备post的数据
        trans_data = {"query":self.trans_str,"from":"zh","to":"en"} if lang== "zh" else             {"query":self.trans_str,"from":"en","to":"zh"}
        #3.发送请求，获取响应
        dict_response = self.parse_url(self.trans_url,trans_data)
        #4.提取翻译的结果
        self.get_ret(dict_response)


if __name__ == ‘__main__‘:
    trans_str= sys.argv[1]
    baidu_fanyi = BaiduFanyi(trans_str)
    baidu_fanyi.run()

# coding=utf-8
import json
import requests
from parse_url import parse_url
from pprint import pprint

url = "https://m.douban.com/rexxar/api/v2/subject_collection/movie_showing/items?start=0&count=18&loc_id=108288"
html_str = parse_url(url)

# json.loads把json字符串转化为python类型
ret1 = json.loads(html_str)
# pprint(ret1)
# print(type(ret1))

# json.dumps能够把python类型转化为json字符串
with open("douban.json","w",encoding="utf-8") as f:
    f.write(json.dumps(ret1,ensure_ascii=False,indent=4))
    # f.write(str(ret1))

# with open("douban.json","r",encoding="utf-8") as f:
#     ret2 = f.read()
#     ret3 = json.loads(ret2)
#     print(ret3)
#     print(type(ret3))


# 使用json。load提取类文件对象中的数据
with open("douban.json","r",encoding="utf-8") as f:
    ret4 = json.load(f)
    print(ret4)
    print(type(ret4))

#json.dump能够把python类型放入类文件对象中
with open("douban1.json","w",encoding="utf-8") as f:
    json.dump(ret1,f,ensure_ascii=False,indent=2)

# coding=utf-8
import re
from parse_url import parse_url
import json

url = "http://36kr.com/"
html_str = parse_url(url)

ret = re.findall("<script>var props=(.*?),locationnal=",html_str)[0]

with open("36kr.json","w",encoding="utf-8") as f:
    f.write(ret)

ret = json.loads(ret)
print(ret)

import requests,sys,json

class Automatic():
    def __init__(self,translade_word):
        self.translade_word = translade_word
        self.langdetect_headers ={"User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Mobile Safari/537.36"}
        self.langdetect_parpams = {"query": translade_word}
        self.langdetect_url ="http://fanyi.baidu.com/langdetect"
        self.translated_url = "http://fanyi.baidu.com/basetrans"

    def langdetect(self):
        response = requests.post(self.langdetect_url,headers = self.langdetect_headers,data=self.langdetect_parpams)
        return json.loads(response.content.decode())[‘lan‘]

    def get_data_language(self,language_word):
        #
        # if "zh" ==language_word:
        #     translade_data ={"query":self.translade_word,
        #                         "from":"zh",
        #                         "to":"en"}
        # else:
        #     translade_data = {"query": self.translade_word,
        #                       "from": language_word,
        #                       "to": "zh"}
        return {"query":self.translade_word,"from":"zh","to":"en"} if "zh" ==language_word                 else {"query": self.translade_word,"from": language_word,"to": "zh"}

    def translade(self,translade_data):
        response = requests.post(self.translated_url,data=translade_data,headers = self.langdetect_headers)
        response_data = json.loads(response.text)
        # print("1111111111",response_data)
        return response_data

    def get_ret(self,response_data):
        data = response_data["trans"][0]["dst"]
        print("{}  翻译后的结果：{}".format(self.translade_word, data))



    def run(self):
        language_word = self.langdetect()
        translade_data= self.get_data_language(language_word)
        response_data = self.translade(translade_data)
        self.get_ret(response_data)

if __name__ == ‘__main__‘:
    translade_word = sys.argv[1]
    automatic = Automatic(translade_word)
    automatic.run()

# coding=utf-8
import requests
from retrying import retry

headers={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}

@retry(stop_max_attempt_number=3)
def _parse_url(url,method,data,proxies):
    print("*"*20)
    if method=="POST":
        response = requests.post(url,data=data,headers=headers,proxies=proxies)
    else:
        response = requests.get(url,headers=headers,timeout=3,proxies=proxies)
    assert  response.status_code == 200
    return response.content.decode()


def parse_url(url,method="GET",data=None,proxies={}):
    try:
        html_str = _parse_url(url,method,data,proxies)
    except:
        html_str = None

    return html_str

if __name__ == ‘__main__‘:
    url = "www.baidu.com"
    print(parse_url(url))

# coding=utf-8
import requests
import json


class DoubanSpider:
    def __init__(self):
        self.url_temp_list = [
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?start={}&count=18&loc_id=108288",
                "country": "US"
            },
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_english_hot/items?start={}&count=18&loc_id=108288",
                "country": "UK"
            },
            {
                "url_temp": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_domestic_hot/items?start={}&count=18&loc_id=108288",
                "country": "CN"
            }
        ]
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36"}

    def parse_url(self, url):  # 发送请求，获取响应
        print(url)
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_content_list(self, json_str):  # 提取是数据
        dict_ret = json.loads(json_str)
        content_list = dict_ret["subject_collection_items"]
        total = dict_ret["total"]
        return content_list, total

    def save_content_list(self, content_list,country):  # 保存
        with open("douban.txt", "a", encoding="utf-8") as f:
            for content in content_list:
                content["country"] = country
                f.write(json.dumps(content, ensure_ascii=False))
                f.write("\n")  # 写入换行符，进行换行
        print("保存成功")

    def run(self):  # 实现主要逻辑
        for url_temp in self.url_temp_list:
            num = 0
            total = 100  # 假设有第一页
            while num < total + 18:
                # 1.start_url
                url = url_temp["url_temp"].format(num)
                # 2.发送请求，获取响应
                json_str = self.parse_url(url)
                # 3.提取是数据
                content_list, total = self.get_content_list(json_str)

                # 4.保存
                self.save_content_list(content_list,url_temp["country"])
                # if len(content_list)<18:
                #     break
                # 5.构造下一页的url地址,进入循环
                num += 18


if __name__ == ‘__main__‘:
    douban_spider = DoubanSpider()
    douban_spider.run()

# coding=utf-8
import requests
import re
import json


class Neihan:
    def __init__(self):
        self.start_url = "http://neihanshequ.com/"
        self.next_url_temp = "http://neihanshequ.com/joke/?is_json=1&app_name=neihanshequ_web&max_time={}"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}

    def parse_url(self, url):  # 发送请求
        print(url)
        response = requests.get(url, headers=self.headers)
        return response.content.decode()

    def get_first_page_content_list(self, html_str):  # 提取第一页的数据
        content_list = re.findall(r"<h1 class=\"title\">.*?<p>(.*?)</p>", html_str, re.S)
        max_time = re.findall("max_time: ‘(.*?)‘,", html_str)[0]
        return content_list, max_time

    def save_content_list(self, content_list):  # 保存
        with open("neihan.txt", "a", encoding="utf-8") as f:
            for content in content_list:
                f.write(json.dumps(content, ensure_ascii=False))
                f.write("\n")
        print("保存成功")

    def get_content_list(self, json_str):  # 提取从第二页开始的json中的数据
        dict_ret = json.loads(json_str)
        data = dict_ret["data"]["data"]
        content_list = [i["group"]["content"] for i in data]
        max_time = dict_ret["data"]["max_time"]
        has_more = dict_ret["data"]["has_more"]
        return content_list, max_time, has_more

    def run(self):  # 实现主要逻辑
        # 1.start_url
        # 2.发送请求，获取响应
        html_str = self.parse_url(self.start_url)
        # 3.提取数据
        content_lsit, max_time = self.get_first_page_content_list(html_str)
        # 4.保存
        self.save_content_list(content_lsit)
        has_more = True  # 有第二页
        while has_more:  # 内涵社区是用has_more 来判断是否有下一页的
            # 5.构造下一页的url地址
            next_url = self.next_url_temp.format(max_time)
            # 6.发送请求，获取响应
            json_str = self.parse_url(next_url)
            # 7.提取数据，提取max_time
            content_lsit, max_time, has_more = self.get_content_list(json_str)
            # 8.保存
            self.save_content_list(content_lsit)
            # 9.循环5-8步


if __name__ == ‘__main__‘:
    neihan = Neihan()
    neihan.run()

# coding=utf-8
from lxml import etree


text = ‘‘‘ <div> <ul> 
        <li class="item-1"><a>first item</a></li> 
        <li class="item-1"><a href="link2.html">second item</a></li> 
        <li class="item-inactive"><a href="link3.html">third item</a></li> 
        <li class="item-1"><a href="link4.html">fourth item</a></li> 
        <li class="item-0"><a href="link5.html">fifth item</a>  
        </ul> </div> ‘‘‘

html = etree.HTML(text)
print(html)
#查看element对象中包含的字符串
# print(etree.tostring(html).decode())

#获取class为item-1 li下的a的herf
ret1 = html.xpath("//li[@class=‘item-1‘]/a/@href")
print(ret1)

#获取class为item-1 li下的a的文本
ret2 = html.xpath("//li[@class=‘item-1‘]/a/text()")
print(ret2)

#每个li是一条新闻，把url和文本组成字典
for href in ret1:
    item = {}
    item["href"] = href
    item["title"] = ret2[ret1.index(href)]
    print(item)

print("*"*100)
#分组，根据li标签进行分组，对每一组继续写xpath
ret3 = html.xpath("//li[@class=‘item-1‘]")
print(ret3)
for i in ret3:
    item=  {}
    item["title"] = i.xpath("a/text()")[0] if len(i.xpath("./a/text()"))>0 else None
    item["href"] = i.xpath("./a/@href")[0] if len( i.xpath("./a/@href"))>0 else None
    print(item)

#  _*_coding: utf-8 _*_
import json
import requests
from parse_url import parse_url
import sys
from pprint import pprint
import re


class douban:
    def __init__(self, url):
        self.url = url
        self.L_url = []
        self.start = 0
        self.html_str = ""
        self.ret = {}

    def get_total(self):
        html_str = parse_url(self.url)
        # json.loads把json字符串转化为python类型
        ret1 = json.loads(html_str)
        total = ret1["total"]
        return total

    def get_url(self, total):
        while self.start < total + 50:
            url = self.url.format(
                self.start + 1, 50)
            self.L_url.append(url)
            self.start += 50

    def get_name(self):
        Wurl = self.url
        reg = r‘https://m.douban.com/rexxar/api/v2/subject_collection/(.*?)/‘
        name = re.findall(reg, Wurl)
        return name[0] + ".json"

    def data(self, name):
        for url in self.L_url:
            self.html_str = parse_url(url)
            ret = json.loads(self.html_str)
            with open(name, "a", encoding="utf-8") as f:
                f.write(json.dumps(ret, ensure_ascii=False, indent=4))

    def run(self):
        total = self.get_total()
        self.get_url(total=total)
        name = self.get_name()
        self.data(name=name)


if __name__ == ‘__main__‘:
    url_dict = {
        "美国": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_american_hot/items?os=android&start=0&count=18&loc_id=108288",
        "英国": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_english_hot/items?os=android&start=0&count=18&loc_id=108288",
        "韩国": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_korean_drama_hot/items?os=android&start=0&count=18&loc_id=108288",
        "中国": "https://m.douban.com/rexxar/api/v2/subject_collection/filter_tv_domestic_hot/items?os=android&start=0&count=18&loc_id=108288"
    }
    Len = len(sys.argv)
    for i in range(Len - 1):
        url_name = sys.argv[i + 1]
        url = url_dict[url_name]
        print(url)
        douban = douban(url)
        douban.run()