链家二手房成交爬虫

def (url):
    try:
        print("processing {}".format(url))
        time.sleep(random.uniform(0.5, 1.6))
        ret = requests.get(url, headers={'User-Agent': ua.chrome}, cookies=get_cookie(), timeout=5)
        return etree.HTML(ret.content)
    except Exception as e:
        return etree.HTML("<None/>")
def get_regions(suffix):
    while True:
        page = spider_get_xml(root_url + suffix)
        hrefs = page.xpath("//div[@data-role='ershoufang']//a")
        regions = {}
        for href in hrefs:
            regions[href.attrib['href']] = href.text
        if len(regions) > 0:
            return regions
def get_section(url):
    while True:
        page = spider_get_xml(root_url + url)
        hrefs = page.xpath("//div[@data-role='ershoufang']/div[2]//a")
        sections = {}
        for href in hrefs:
            sections[href.attrib['href']] = href.text
        if len(sections) > 0:
            return sections

def get_cookie():
    with open('cookie', 'r') as f:
        cookies = {}
        for line in f.read().split(';'):
            name, value = line.strip().split('=', 1)  
            cookies[name] = value
        return cookies大专栏  链家二手房成交爬虫v>

def process_traded_section(url):
    while True:
        page = spider_get_xml(root_url + url)
        data = page.xpath("//*[@class='total fl']/span")
        if len(data) > 0:
            total_num = int(data[0].text)
            page_num = int(math.ceil(total_num / 30))
            # print('{0} has {1} house(s)'.format(section_name, total_num))
            break
    for i in range(1, page_num + 1):
        process_traded_page(root_url + url + "pg{0}/".format(i))
def process_traded_page(url):
    # record_url(url)
    # return
    index = 16
    while index > 0:
        page = spider_get_xml(url)
        data = page.xpath("//*[@class='total fl']/span")
        if len(data) > 0:
            break
        index -= 1
        open_browser(url)
        input("看看弹出的浏览器信息，处理之后按任意键继续爬")
    if index == 0:
        print("get stuck in page {}", url)
        return
    house_refs = page.xpath("//ul[@class='listContent']/li")
    for house_ref in house_refs:
        process_traded_house(house_ref)
def process_traded_house(house_ref):
    try:
        title = house_ref.xpath("div[@class='info']/div[@class='title']/a")[0]
        house_url = title.attrib['href']
        xiaoqu, huxing, area = title.text.split(' ')
        area = area[:-2]
        deal_date = house_ref.xpath(".//div[@class='dealDate']")[0].text
        price = float(house_ref.xpath(".//div[@class='totalPrice']/span")[0].text)
        unit_price = float(house_ref.xpath(".//div[@class='unitPrice']/span")[0].text)
        info = "{0}t{1:g}t{2:g}t{3}t{4}t{5}t{6}t{7}".format(house_url, price, unit_price, huxing, area,
                                                                   current_section_name, xiaoqu, deal_date)
        record_data(info)
    except Exception as e:
        pass