码迷,mamicode.com
首页 > 其他好文 > 详细

获取安居客小区信息

时间:2017-08-07 20:48:18      阅读:408      评论:0      收藏:0      [点我收藏+]

标签:imp   net   ring   taf   text   guid   tar   center   address   

# -*- coding: utf-8 -*-
"""
Created on Sat Jun 24 22:03:17 2017

@author: willowj
"""
import sys
stdout, stdin, stderr  =  sys.stdout, sys.stdin, sys.stderr
reload(sys)
sys.stdout, sys.stdin, sys.stderr =  stdout, stdin, stderr
sys.setdefaultencoding(utf8)  


import requests
#import codecs
import pandas as pd 
import numpy as np
from lxml import html
import math
import time
import re
#安居客 小区

#广州

    
        
headers = {
accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8,
accept-encoding:gzip, deflate, sdch,
accept-language:zh-CN,zh;q=0.8,en;q=0.6,
cache-control:max-age=0,
#‘cookie‘:‘als=0; ctid=12; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1498312808; sessid=6D343DE2-F344-C0B1-A8FD-794CF9851F6B; lps=http%3A%2F%2Fguangzhou.anjuke.com%2Fcommunity%2Fview%2F853712%7C; _ga=GA1.2.1457521041.1498312777; _gid=GA1.2.341883465.1498312777; aQQ_ajkguid=5E03C75B-5FDF-6879-5AF9-5DDAB323E51F; twe=2; 58tj_uuid=632d5e6b-78a6-483e-9c41-7e2e21023e74; new_session=0; init_refer=; new_uv=8‘,
referer:https://guangzhou.anjuke.com/community/,
Upgrade-Insecure-Requests:"1",
#"Connection": "close",
"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
        }

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"
        ]

def not_nulist_first(list_):
    if not list_:
        return None
    else:
        return list_[0]

def get_comnuities(page_response_):
    #从网页解析小区数据
    print page_response_.url, getting
    etree = html.document_fromstring(page_response_.text)
    communities = etree.xpath(//div[@_soj="xqlb"])
    
    #for community in communities:
    page_communities1 = []
    for community in communities:
    
        name = community.xpath("./a/@title")[0]

        community_web = html.urljoin(init_url, community.xpath("./a/@href")[0])
        address = community.xpath(".//address/text()")[0].strip()
        established_date = community.xpath(.//p[@class="date"]/text())[0]
        price_perm2 = community.xpath(.//strong/text())
        price_perm2 = not_nulist_first(price_perm2)

        page_communities1.append([name, price_perm2, established_date, address, community_web])
    print name
    return page_communities1   
    


class AnjukeCommunity(object):
 """docstring for ClassName"""
    def __init__(self, init_url=None):
        #广州安居客
        self.init_url = https://guangzhou.anjuke.com/community/
        self.requ = requests.session()
        self.requ.adapters.DEFAULT_RETRIES = 5
        self.pages_max = self.pages_max()

    def pages_max(self):
        #最大页数
        page_response = self.requ.get(self.init_url, headers=headers)
        init_etree = html.document_fromstring(page_response.text)
        community_Nums = int(init_etree.xpath(//span[@class="tit"]/em/text())[-1])
        communities = init_etree.xpath(//div[@_soj="xqlb"])
        return int(math.ceil(float(community_Nums)/len(communities)))       

    def get_all_comnunities(self):

        page_communities = []

        for page in range(1, pages_max+1):
            url_ = %s/p%s/%(self.init_url, page)
            #打开网页
            headers["User-Agent"] = np.random.choice(USER_AGENTS)
            page_response_ = self.requ.get(url_, headers=headers)
            print  page_response_.url, page_response_.status_code, start
            time.sleep(20)
            page_response_.close()
            page_response_.connection.close()

            #解析网页数据
            communi = get_comnuities(page_response_)
            page_communities.extend(communi)


#网页标题
print init_etree.xpath(//div[@_soj="xqlb"]/a/@title)[0]


init_url = https://guangzhou.anjuke.com/community/
Anjuke_guangzhou = AnjukeCommunity(init_url)
page_communities = Anjuke_guangzhou.get_all_comnunities()

#pandas 规整数据    
communities_pd = pd.DataFrame(page_communities, columns=[name, price_perm2, established_date, address, community_web])
communities_pd[price_perm2] = communities_pd[price_perm2].astype(float)
communities_pd[price_part] = communities_pd[price_perm2]//5000 * 5000
communities_pd[established_year] = communities_pd[established_date].str.extract((\d+),expand=False).astype(int) 
#communities_pd[‘religion‘] = communities_pd[‘address‘].str.extract(u"[(.+?)-", expand=False)
communities_pd[religion] = communities_pd[address].str.slice(1,3)

cols = [religion, name, price_part, price_perm2, established_year, established_date, address, community_web]

communities_pd[cols].to_excel(u"安居客广州小区.xlsx",  index=False,  encoding=gb18030)    
 

 

获取安居客小区信息

标签:imp   net   ring   taf   text   guid   tar   center   address   

原文地址:http://www.cnblogs.com/willowj/p/7300812.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!