Python 爬取 房天下

import requests
from requests import ConnectionError
from bs4 import BeautifulSoup
import pymongo

#MongoDB的配置
MONGO_URL = ‘localhost‘
MONGO_DB = ‘Fangtianxia‘
MONGO_TABLE = ‘HouseInfo‘
#配置MongoDB
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]

base_url = ‘http://esf.sz.fang.com/housing/__1_0_0_0_‘
headers = {
    ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36‘
}

#发送请求，获得响应
def get_html(url):
    print(‘正在爬取‘,url)
    try:
        response = requests.get(url=url,headers=headers)
        if response.status_code == 200:
            return response.text
        else:
            print(response.status_code)
            return None
    except ConnectionError:
        print(‘出现错误！‘)
        return None

#解析响应体，进行信息提取
def parser_html(html):
    soup = BeautifulSoup(html,‘lxml‘)
    names = soup.select(‘.houseList .list .plotListwrap dd p‘)[::3]#小区名称、小区链接、
    addresses = soup.select(‘.houseList .list .plotListwrap dd p‘)[1::3]  # 小区地址
    years = soup.select(‘.houseList .list .plotListwrap dd li‘)[::3]#修建时间
    princes = soup.select(‘.houseList .list .listRiconwrap .priceAverage‘)#小区价格
    ratioes = soup.select(‘.houseList .list .listRiconwrap .ratio‘)#小区涨幅
    for i in range(len(names)):
        house ={
            ‘name‘ : names[i].a.text.strip(),
            ‘url‘: names[i].find(name=‘a‘).attrs[‘href‘].strip(),
            ‘address‘:addresses[i].text.strip(),
            ‘prince‘:princes[i].text.strip()[:-4],
            ‘ratio‘: ratioes[i].text.strip()
        }
        save_to_mongo(house)


def save_to_mongo(data):
    try:
        if db[MONGO_TABLE].insert(data):
            print(‘成功保存到数据库‘,data)
    except Exception:
        print(‘保存失败！‘,data)

#主体函数
def main():
    for page in range(1,101):
        url  = base_url + str(page) +‘_0_0_0/‘#深圳的小区房价，不同页码下的url
        html = get_html(url)#完成请求，获得响应体
        parser_html(html)#解析响应体，提取感兴趣的信息

if __name__==‘__main__‘:
    main()