亚马逊美国Lightning_Deals爬虫

import csv
import json
import time
import redis
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from googletrans import Translator


# import requests.packages.urllib3.util.ssl_
# requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS = ‘ALL‘

def trademark(goods_brand):
    if goods_brand == None:
        brand_register = ‘未知‘
        return brand_register
    if goods_brand == ‘null‘:
        brand_register = ‘未知‘
        return brand_register
    r = redis.Redis(host=‘127.0.0.1‘, port=6379, db=0, decode_responses=True, password=XXXXX)
    redis_brand = r.get(goods_brand)
    if redis_brand == ‘Registered‘:
        return redis_brand
    if redis_brand == None or redis_brand == "unregistered":
        ff_option = Options()
        ff_option.add_argument(‘-headless‘)
        browser = webdriver.Chrome(‘C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe‘, options=ff_option)
        browser.get(‘http://tmsearch.uspto.gov‘)
        browser.find_element_by_xpath(‘/html/body/center/table[1]/tbody/tr[2]/td/font/font/a‘).click()
        # 向文本框中传入待查询的品牌
        browser.find_element_by_name("p_s_PARA2").send_keys(goods_brand)
        # 点击查询按钮
        browser.find_element_by_xpath("//input[@onclick=‘changeCurlyQuote();‘]").click()
        # 找到查询商标结果页面的title
        # 如果查询结果页的title为“TESS -- Error”则判定改商标未注册
        # 已注册用True表示，未注册用False表示
        register_html = browser.page_source
        if len(register_html) > 920:
            brand_register = ‘Registered‘
        else:
            brand_register = ‘unregistered‘
        r.set(goods_brand, brand_register)
        browser.quit()
        return brand_register



def send_request(url, headers, proxies, session):
    flag = False
    while not flag:
        try:
            response = session.get(url, headers=headers, proxies=proxies, verify=False)
            return response
        except Exception as E:
            print(E)
            print(‘失败，正在重新尝试。‘)
            continue
        flag = True


def rank(goods_soup):
    goods_rank_li = goods_soup.find(‘li‘, id=‘SalesRank‘)
    if goods_rank_li:
        goods_rank = goods_rank_li.text.strip().replace("\n", ‘‘).replace(
            ".zg_hrsr { margin: 0; padding: 0; list-style-type: none; }.zg_hrsr_item { margin: 0 0 0 10px; }.zg_hrsr_rank { display: inline-block; width: 80px; text-align: right; }",
            "").replace("Amazon Best Sellers Rank:", ‘‘).replace("Amazon Bestsellers Rank: ", "").replace("Best Sellers Rank", "").strip().replace("#", ‘\n#‘)
        return goods_rank
    else:
        goods_rank_table = goods_soup.find(‘table‘, id=‘productDetails_detailBullets_sections1‘)
        if goods_rank_table:
            goods_rank_table_tr = goods_rank_table.find_all(‘tr‘)
            for tr in goods_rank_table_tr:
                if tr.find(‘th‘).text.strip().replace(" ", ‘‘) == ‘BestSellersRank‘:
                    goods_rank = tr.find(‘td‘).text.replace("\n", ‘‘).replace(
                        ".zg_hrsr { margin: 0; padding: 0; list-style-type: none; }.zg_hrsr_item { margin: 0 0 0 10px; }.zg_hrsr_rank { display: inline-block; width: 80px; text-align: right; }",
                        "").replace("Amazon Best Sellers Rank:", ‘‘).replace("Amazon Bestsellers Rank: ", "").replace("Best Sellers Rank", "").strip().replace("#", ‘\n#‘)
                    return goods_rank


def title(goods_soup):
    goods_title_span = goods_soup.find(‘span‘, id=‘productTitle‘)
    if goods_title_span:
        goods_title = goods_title_span.text.strip().replace("\xa0", ‘‘).replace(",", ‘‘)
        return goods_title
    else:
        goods_title = ‘null‘
        return goods_title


def brand(goods_soup):
    goods_brand_a = goods_soup.find(‘a‘, id=‘bylineInfo‘)
    if goods_brand_a:
        goods_brand = goods_brand_a.text.strip().replace("\xa0", ‘‘)
        return goods_brand
    else:
        try:
            goods_brand = goods_soup.find(‘a‘, id=‘brand‘).text.strip().replace("\xa0", ‘‘)
            return goods_brand
        except AttributeError:
            goods_brand = ‘null‘
            return goods_brand


def star(goods_soup):
    goods_star_span = goods_soup.find(‘span‘, id=‘acrPopover‘)
    if goods_star_span:
        goods_star = goods_star_span.find(‘i‘).text.split(" out of 5 stars")[0]
        return goods_star
    else:
        goods_star = ‘null‘
        return goods_star


def review(goods_soup):
    goods_review_sapn = goods_soup.find(‘span‘, id=‘acrCustomerReviewText‘)
    if goods_review_sapn:
        goods_review = goods_review_sapn.text.split(" customer reviews")[0]
        return goods_review
    else:
        goods_review = ‘null‘
        return goods_review


def price(goods_soup):
    goods_price_span = goods_soup.find(‘span‘, id=‘priceblock_dealprice‘)
    if goods_price_span:
        goods_price = goods_price_span.text.replace("$", ‘‘)
        return goods_price
    else:
        goods_price_span = goods_soup.find(‘span‘, id=‘newBuyBoxPrice‘)
        if goods_price_span:
            goods_price = goods_price_span.text.replace("$", ‘‘)
            return goods_price
        else:
            goods_price_sale = goods_soup.find(‘span‘, id=‘priceblock_saleprice‘)
            if goods_price_sale:
                goods_price = goods_price_sale.text.replace("$", ‘‘)
                return goods_price
            else:
                try:
                    goods_price = goods_soup.find(‘span‘, id=‘priceblock_ourprice‘).text.replace("$", ‘‘)
                    return goods_price
                except AttributeError:
                    goods_price = ‘null‘
                    return goods_price


def date(goods_soup):
    goods_date_div = goods_soup.find(‘div‘, id=‘detailBullets_feature_div‘)
    if goods_date_div:
        goods_date_all_li = goods_date_div.find_all(‘li‘)
        for li in goods_date_all_li:
            li_title_span = li.find(‘span‘, class_=‘a-text-bold‘)
            if li_title_span:
                li_title = li_title_span.text.strip()

                if li_title == ‘Date first listed on Amazon:‘:
                    goods_date = li.text.strip().replace("Date first listed on Amazon:", ‘‘).strip()
                    return goods_date
    else:
        goods_date_table = goods_soup.find(‘table‘, id=‘productDetails_detailBullets_sections1‘)
        if goods_date_table:
            goods_date_table_tr = goods_date_table.find_all(‘tr‘)
            for tr in goods_date_table_tr:
                if tr.find(‘th‘).text.strip().replace(" ", ‘‘) == ‘DatefirstlistedonAmazon‘:
                    goods_date = tr.find(‘td‘).text.replace("\n", ‘‘).strip()
                    return goods_date


def translator(title):
    translator = Translator(service_urls=[‘translate.google.cn‘])
    after_title = translator.translate(‘%s‘ % title, src=‘de‘, dest="zh-CN")
    translation_title = str(after_title).split("text=")[1].replace(", pronunciation=None)", ‘‘)
    return translation_title


def run(page):
    print("当前页码为：%s" % page)
    # url = ‘http://webapi.http.zhimacangku.com/getip?num=1&type=1&pro=&city=0&yys=0&port=11&time=1&ts=0&ys=0&cs=0&lb=4&sb=0&pb=4&mr=1&regions=‘
    # proxy = requests.get(url).text.replace("\n", ‘‘)
    # proxies = {
    #     "http": "http://" + proxy,
    #     ‘https‘: ‘https://‘ + proxy
    # }
    base_url = ‘https://www.amazon.com/dp/‘
    headers = {
        ‘accept-encoding‘: ‘gzip, deflate, br‘,
        # ‘accept-language‘: ‘zh-CN,zh;q=0.9‘,
        ‘upgrade-insecure-requests‘: ‘1‘,
        ‘user-agent‘: "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36",
        ‘accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8‘,
        ‘cache-control‘: ‘max-age=0‘,
        ‘authority‘: ‘www.amazon.com‘,
        # ‘cookie‘: ‘session-id=140-0436092-5114916; session-id-time=2082787201l; ubid-main=134-7819260-0954509; x-wl-uid=1D2qwOkkELPrF8q/YqteGc9JYBV5ARtF3Mc2jW/s32idPGnjnZ7sbB5wHyMnR/u9Sw34fPkbO2xA=; session-token=1c+rOmuhW1M6euftwY4+w/swVPHn3AudZXzHSqu/xcF4uMyS946ZcjCM2If+kp/T4sOC1KoBEXBODEkbBGmd9AbT7XagZlu0xFW9tti1p/z0xCUCzs5/GOrqAi7knU6259ewXpjCRgotqeM8IgNTqV1AXCu/yCq/9abumda60iIUqCTSnaleSMeEU1l25LK0Y4FnXzmTNEgJJLvLtbgLF6Hnw7uyO2qTA2xB7uIB/ZZlC+TvrnOtd3cGa7jV6MHc; s_nr=1526623968134-New; s_vnum=1958623968135^%^26vn^%^3D1; s_dslv=1526623968135; lc-main=en_US; x-amz-captcha-1=1527334802791799; x-amz-captcha-2=kU9AAR92z09BFOUgsoQgXw==; skin=noskin; csm-hit=tb:SDXEVFJG2SJZZRQZDRAF+s-SDXEVFJG2SJZZRQZDRAF^|1527472665005^&adb:adblk_no‘,
    }

    ff_option = Options()
    ff_option.add_argument(‘-headless‘)
    browser = webdriver.Chrome(‘C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe‘, chrome_options=ff_option)
    browser.get(‘https://www.amazon.com/gp/goldbox/ref=gbps_ftr_s-4_d724_page_‘ + str(
        page) + ‘?gb_f_deals1=dealStates:AVAILABLE%252CWAITLIST%252CWAITLISTFULL%252CEXPIRED%252CSOLDOUT%252CUPCOMING,page:‘ + str(page) + ‘,dealTypes:LIGHTNING_DEAL,dealsPerPage:48‘)
    time.sleep(10)
    pageSource = browser.page_source
    page_soup = BeautifulSoup(pageSource, ‘lxml‘)
    all_goods_div = page_soup.find(‘div‘, id=‘widgetContent‘).find_all(‘div‘, class_=‘a-section dealContainer‘)
    print(len(all_goods_div))
    for goods_div in all_goods_div:
        session = requests.session()
        # 给每个商品生成一个空列表
        goods_info_list = []
        schedule_div = goods_div.find(‘div‘, ‘a-column a-span5 a-text-left unitLineHeight‘)

        # 进度条
        if schedule_div:
            schedule = schedule_div.find(‘div‘, ‘a-row unitLineHeight‘).text.strip().replace(" Claimed", ‘‘).replace("\xa0", ‘‘).replace("\xae", ‘‘).replace("\u2122", ‘‘)
        else:
            schedule = ‘null‘

        # 距离结束
        timer = goods_div.find(‘span‘, role=‘timer‘)
        if timer:
            end_time = timer.text.strip().replace("\xa0", ‘‘).replace("\xae", ‘‘).replace("\u2122", ‘‘)
        else:
            end_time = ‘null‘
        now_time = time.strftime(‘%H:%M:%S‘, time.localtime(time.time()))

        dealtitle = goods_div.find(‘a‘, id=‘dealImage‘)
        # 取到商品详情页url
        try:
            goods_asin = dealtitle[‘href‘].split("dp/")[1].split(‘/‘)[0]
        except IndexError:
            continue
        goods_url = base_url + goods_asin
        goods_html = requests.get(goods_url, headers=headers)
        goods_soup = BeautifulSoup(goods_html.text, ‘lxml‘)
        print("商品链接为：" + goods_url)
        goods_asin = goods_url.split(‘dp/‘)[1]
        goods_title = title(goods_soup)
        after_title = translator(goods_title)
        goods_brand = brand(goods_soup)
        goods_star = star(goods_soup)
        goods_review = review(goods_soup)
        goods_price = price(goods_soup)
        goods_rank = rank(goods_soup)
        goods_date = date(goods_soup)
        brand_register = trademark(goods_brand)
        if goods_date == None:
            goods_date = ‘null‘
        print("schedule:" + schedule)
        print("goods_title:" + goods_title)
        print("after_title:" + after_title)
        print("goods_asin:" + goods_asin)
        print("goods_brand:" + goods_brand)
        print("brand_register:" + brand_register)
        print("goods_date:" + str(goods_date))
        print("goods_star:" + goods_star)
        print("end_time:" + end_time)
        print("now_time:" + now_time)
        print("goods_review:" + goods_review)
        print("goods_rank:" + str(goods_rank))
        print("goods_price:" + goods_price)
        goods_info_list.append(schedule)
        goods_info_list.append(end_time)
        goods_info_list.append(now_time)
        goods_info_list.append(goods_title)
        goods_info_list.append(after_title)
        goods_info_list.append(goods_brand)
        goods_info_list.append(brand_register)
        goods_info_list.append(goods_asin)
        goods_info_list.append(goods_date)
        goods_info_list.append(goods_star)
        goods_info_list.append(goods_review)
        goods_info_list.append(goods_rank)
        goods_info_list.append(goods_price)
        print(‘=========================================‘)
        csvFile = open(‘./Lightning_Deals_US/Lightning_Deals_US_%s.csv‘ % str(page), ‘a‘, newline=‘‘, encoding=‘gb18030‘)  # 设置newline，否则两行之间会空一行
        writer = csv.writer(csvFile)
        writer.writerow(goods_info_list)
        csvFile.close()


if __name__ == ‘__main__‘:
    for i in range(1, 22):
        run(i)