读<<为了替女朋友买件内衣，用Python爬虫>>有感

def get_driver(executable_path):
    options = webdriver.ChromeOptions()
    options.add_argument(‘lang=zh_CN.UTF-8‘)
    options.add_argument("–disable-javascript")
    options.add_argument("‘user-agent‘=‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36‘")
    prefs= {
    "profile.managed_default_content_settings.images":2,
    "profile.content_settings.plugin_whitelist.adobe-flash-player":2,
    "profile.content_settings.exceptions.plugins.*,*.per_resource.adobe-flash-player":2,
}
    options.add_experimental_option("prefs", prefs)
    #这个才是重点，貌似不设置这个的话，selenium会直接被ALI识别出来，
    options.add_experimental_option("excludeSwitches",["enable-automation"])
    driver = webdriver.Chrome(chrome_options=options,executable_path=executable_path)
    driver.get("https://login.taobao.com")
    time.sleep(60)
    print("登录成功。。。")
    pickle.dump(driver.get_cookies(), open("cookies.pkl", "wb"))
    cookies_list = pickle.load(open("cookies.pkl", "rb"))
    for cookie in cookies_list:
        cookie_dict = {
            "domain": "tmall.com",  # 火狐浏览器不用填写，谷歌要需要
            ‘name‘: cookie.get(‘name‘),
            ‘value‘: cookie.get(‘value‘),
            "expires": "",
            ‘path‘: ‘/‘,
            ‘httpOnly‘: False,
            ‘HostOnly‘: False,
            ‘Secure‘: False}
        driver.add_cookie(cookie_dict)
    return driver

driver.get("http://list.tmall.com/search_product.htm?cat=50095659")
data_summary = []
elements = driver.find_elements_by_class_name("product-iWrap")
for element in elements:
    item_href = element.find_element_by_class_name("productImg").get_attribute("href")
    #item_img = element.find_element_by_tag_name("img").get_attribute("src")
    item_img = re.search("//(img.*.jpg)",element.find_element_by_tag_name("img").get_attribute(‘outerHTML‘)).groups()[0]
    item_price = element.find_element_by_xpath("./p[@class=‘productPrice‘]/em").get_attribute("title")
    item_title = element.find_element_by_xpath("./p[@class=‘productTitle‘]/a").get_attribute("title")
    store_href = element.find_element_by_xpath("./div[@class=‘productShop‘]/a[@class=‘productShop-name‘]").get_attribute("href")
    try:
        store_name = element.find_element_by_xpath("./p[@class=‘productStatus‘]/span[3]").get_attribute("data-nick")
        item_quantities = element.find_element_by_xpath("./p[@class=‘productStatus‘]/span").text
        comments_num = element.find_element_by_xpath("./p[@class=‘productStatus‘]/span[2]").text
        comments_herf = element.find_element_by_xpath("./p[@class=‘productStatus‘]/span[2]/a").get_attribute("href")
    except NoSuchElementException:
        store_name = element.find_element_by_xpath("./div[@class=‘productShop‘]/a[@class=‘productShop-name‘]").text
        item_quantities = "null"
        comments_num = "null"
        comments_herf = "null"
    data_summary.append([item_href,item_img,item_price,item_title,store_href,store_name,item_quantities,comments_num,comments_herf])

def swipe_down(driver,second):
    #先控制好翻页的动作
    for i in range(int(second/0.2)):
        #根据i的值，模拟上下滑动
        if(i%2==0):
            js = ‘window.scrollBy(0,{})‘.format(30+50*i)
        else:
            js = ‘window.scrollBy(0,{})‘.format(40 * i)
        driver.execute_script(js)
        time.sleep(np.random.randint(1,3)*0.2)

def get_comment(driver,url,page=100):
    driver.get(url)
    time.sleep(3)
    try:
        WebDriverWait(driver, 20, 10).until(EC.presence_of_element_located((By.XPATH, ‘//*[@id="J_TabBar"]/li[2]/a‘))).click()
    except TimeoutException:
        driver.refresh()
        swipe_down(driver, random.randint(2, 4))
        WebDriverWait(driver, 20, 15).until(EC.presence_of_element_located((By.XPATH, ‘//*[@id="J_TabBar"]/li[2]/a‘))).click()
    except ElementClickInterceptedException:
        point = driver.find_element_by_xpath(‘//*[@id="J_TabBar"]/li[2]/a‘)
        ActionChains(driver).move_to_element(point).perform()
        time.sleep(2)
        driver.execute_script("arguments[0].click();", point)
    comments_list = []
    comments_list.append(url)
    page_num = 0
    for i in range(page):
        swipe_down(driver, random.randint(1, 4))
        comment_list = []
        for i in driver.find_elements_by_xpath(‘//*[@id="J_Reviews"]/div/div[6]/table/tbody/tr‘):
            comment_list.append(i.text)
        comments_list.append(comment_list)
        button = None
        try:
            button = driver.find_element_by_xpath(‘//div[@class="rate-paginator"]/a[text()= "下一页>>"]‘)
        except NoSuchElementException:
            print("没找到，是不是页数不到{}".format(i))
            break
        driver.execute_script("arguments[0].scrollIntoView();", button)
        driver.execute_script(‘window.scrollBy(0,{})‘.format(np.random.randint(1,4)*(-200)))
        ActionChains(driver).move_to_element(button).perform()
        driver.execute_script("arguments[0].click();", button)
    return comments_list

def split_comment(row):
    row = re.sub("解释：.*\\n","",row)
    row = re.sub("商品：","",row)
    row = re.sub("\\n服务：","",row)
    if row.startswith("初次评价:\n"):
        pattern = "^初次评价:\\n(.*)\\n(.*)\\n收货([\d\S天后]*)追加：\\n(.*)[\\n\S]*\\n颜色分类：(.*)\\n尺码：(.*)\\n(.*)[\\n]?(.*)"
    else:
        pattern = "(.*)\\n(.*)\\n颜色分类：(.*)\\n尺码：(.*)\\n(.*)[\\n]{0,1}(.*)"
    splited_list = re.match(pattern,row).groups()
    return [i for i in splited_list]

1.你只是长的像吴彦祖，你的内心是郭靖，是狄云；
2.变通；
3.你还是那个在消防演练时，大喊晚辈明教教主张无忌，如果各位信得过在下的就跳下来的帅小伙。