标签:imp net ring taf text guid tar center address
# -*- coding: utf-8 -*- """ Created on Sat Jun 24 22:03:17 2017 @author: willowj """ import sys stdout, stdin, stderr = sys.stdout, sys.stdin, sys.stderr reload(sys) sys.stdout, sys.stdin, sys.stderr = stdout, stdin, stderr sys.setdefaultencoding(‘utf8‘) import requests #import codecs import pandas as pd import numpy as np from lxml import html import math import time import re #安居客 小区 #广州 headers = { ‘accept‘:‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8‘, ‘accept-encoding‘:‘gzip, deflate, sdch‘, ‘accept-language‘:‘zh-CN,zh;q=0.8,en;q=0.6‘, ‘cache-control‘:‘max-age=0‘, #‘cookie‘:‘als=0; ctid=12; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1498312808; sessid=6D343DE2-F344-C0B1-A8FD-794CF9851F6B; lps=http%3A%2F%2Fguangzhou.anjuke.com%2Fcommunity%2Fview%2F853712%7C; _ga=GA1.2.1457521041.1498312777; _gid=GA1.2.341883465.1498312777; aQQ_ajkguid=5E03C75B-5FDF-6879-5AF9-5DDAB323E51F; twe=2; 58tj_uuid=632d5e6b-78a6-483e-9c41-7e2e21023e74; new_session=0; init_refer=; new_uv=8‘, ‘referer‘:‘https://guangzhou.anjuke.com/community/‘, ‘Upgrade-Insecure-Requests‘:"1", #"Connection": "close", "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" } USER_AGENTS = [ "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)", "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)", "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1", "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0", "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36" ] def not_nulist_first(list_): if not list_: return None else: return list_[0] def get_comnuities(page_response_): #从网页解析小区数据 print page_response_.url, ‘getting‘ etree = html.document_fromstring(page_response_.text) communities = etree.xpath(‘//div[@_soj="xqlb"]‘) #for community in communities: page_communities1 = [] for community in communities: name = community.xpath("./a/@title")[0] community_web = html.urljoin(init_url, community.xpath("./a/@href")[0]) address = community.xpath(".//address/text()")[0].strip() established_date = community.xpath(‘.//p[@class="date"]/text()‘)[0] price_perm2 = community.xpath(‘.//strong/text()‘) price_perm2 = not_nulist_first(price_perm2) page_communities1.append([name, price_perm2, established_date, address, community_web]) print name return page_communities1 class AnjukeCommunity(object): """docstring for ClassName""" def __init__(self, init_url=None): #广州安居客 self.init_url = ‘https://guangzhou.anjuke.com/community/‘ self.requ = requests.session() self.requ.adapters.DEFAULT_RETRIES = 5 self.pages_max = self.pages_max() def pages_max(self): #最大页数 page_response = self.requ.get(self.init_url, headers=headers) init_etree = html.document_fromstring(page_response.text) community_Nums = int(init_etree.xpath(‘//span[@class="tit"]/em/text()‘)[-1]) communities = init_etree.xpath(‘//div[@_soj="xqlb"]‘) return int(math.ceil(float(community_Nums)/len(communities))) def get_all_comnunities(self): page_communities = [] for page in range(1, pages_max+1): url_ = ‘%s/p%s/‘%(self.init_url, page) #打开网页 headers["User-Agent"] = np.random.choice(USER_AGENTS) page_response_ = self.requ.get(url_, headers=headers) print page_response_.url, page_response_.status_code, ‘start‘ time.sleep(20) page_response_.close() page_response_.connection.close() #解析网页数据 communi = get_comnuities(page_response_) page_communities.extend(communi) #网页标题 print init_etree.xpath(‘//div[@_soj="xqlb"]/a/@title‘)[0] init_url = ‘https://guangzhou.anjuke.com/community/‘ Anjuke_guangzhou = AnjukeCommunity(init_url) page_communities = Anjuke_guangzhou.get_all_comnunities() #pandas 规整数据 communities_pd = pd.DataFrame(page_communities, columns=[‘name‘, ‘price_perm2‘, ‘established_date‘, ‘address‘, ‘community_web‘]) communities_pd[‘price_perm2‘] = communities_pd[‘price_perm2‘].astype(float) communities_pd[‘price_part‘] = communities_pd[‘price_perm2‘]//5000 * 5000 communities_pd[‘established_year‘] = communities_pd[‘established_date‘].str.extract(‘(\d+)‘,expand=False).astype(int) #communities_pd[‘religion‘] = communities_pd[‘address‘].str.extract(u"[(.+?)-", expand=False) communities_pd[‘religion‘] = communities_pd[‘address‘].str.slice(1,3) cols = [‘religion‘, ‘name‘, ‘price_part‘, ‘price_perm2‘, ‘established_year‘, ‘established_date‘, ‘address‘, ‘community_web‘] communities_pd[cols].to_excel(u"安居客广州小区.xlsx", index=False, encoding=‘gb18030‘)
标签:imp net ring taf text guid tar center address
原文地址:http://www.cnblogs.com/willowj/p/7300812.html