赶集租房

#!/usr/bin/env python
# -*- coding:utf-8 -*-

# Author : zhibo.wang
# E-mail : d_1206@qq.com
# Date   : 18/04/24 09:22:58
# Desc   : 赶集租房

import hashlib
import socket
import time
import scrapy
import copy
from house_spider.agent import get_agent
from bs4 import BeautifulSoup
from utils.time_convert import get_time_stamp
from utils.conmongodb import mongo_con_keepalive


class GanJiSpider(scrapy.Spider):
    name = ‘ganji_rent_spider‘
    allowed_domains = [‘ganji.com‘]
    custom_settings = {
        "DOWNLOAD_DELAY": 0.4,
    }

    def __init__(self):
        self.db = mongo_con_keepalive()
        self.start_data = [{"cityname": i.get("cityname"),
                            "city_code": i.get("ganji_code"),
                            "url": "http://{0}.ganji.com/fang1/m1/".format(i.get("ganji_code"))}
                           for i in self.db.get_collection("para_city_list").find({}, {"_id":0})]

        self.efs = self.db.get_collection(‘pathdir_dict‘)
        self.ganji_header = get_agent(‘ganji_header‘)
        self.path_dir = "realstate/house/ganji/rent/{0}/".format(get_time_stamp())
        self.efs.insert_one({‘pathdir‘: self.path_dir, ‘website‘: ‘ganji_rent‘, ‘flag‘: False})


        super(GanJiSpider, self).__init__()

    def start_requests(self):
        reqs = []
        for start_ in self.start_data:
            hd = copy.deepcopy(self.ganji_header[0])
            url = start_.get("url")
            hd[‘Host‘] = url.split("/")[2]
            data = {"cityname": start_.get("cityname"), "city_code": start_.get("city_code"), "url": url, "host": hd[‘Host‘]}
            reqs.append(scrapy.Request(url, headers=hd, callback=self.parse_district, meta=data))

        return reqs

    def parse_district(self, response):
        hd = copy.deepcopy(self.ganji_header[0])
        host = response.request.meta.get("host")
        cityname = response.request.meta.get("cityname")
        city_code = response.request.meta.get("city_code")
        hd[‘Host‘] = host
        hd["Referer"] = response.request.meta.get("url")
        soup = BeautifulSoup(response.text, ‘html.parser‘)
        district_list = soup.find("div", class_="thr-list").find_all("li", class_="item")
        for district_ in district_list:

            url = "http://{0}.ganji.com{1}".format(city_code, district_.find("a").get("href"))
            data = {"cityname": cityname, "city_code": city_code,"url": url, "host": hd[‘Host‘], "page": True}
            yield scrapy.Request(url, headers=hd, callback=self.parse_page, meta=data)



    def parse_page(self, response):
        # 处理列表页

        hd = copy.deepcopy(self.ganji_header[0])
        host = response.request.meta.get("host")
        cityname = response.request.meta.get("cityname")
        hd[‘Host‘] = host
        hd["Referer"] = response.request.meta.get("url")
        soup = BeautifulSoup(response.text, ‘html.parser‘)
        city_code = response.request.meta.get("city_code")

        try:
            house_list = soup.find("div", class_="f-list js-tips-list").find_all("div", class_="f-list-item ershoufang-list")
            # 获取列表页 所有房源地址
            house_datas = [
                "http://{0}.ganji.com{1}".format(city_code, i.get("href"))
                for i in house_list
            ]
            for house_data in house_datas:
                yield scrapy.Request(house_data, headers=hd, callback=self.parse_item)
        except:
            print("没有租房数据")

        if response.request.meta.get("page"):
            try:
                all_count = int(
                    soup.find("p", class_="m-result f-fr").find("span", class_="num").get_text().replace("套", ""))
                end_apge = int(all_count/50)
                if all_count % 50 != 0:
                    end_apge += 1
                page_urls = [
                    response.request.meta.get("url").replace("m1/", "m1o{0}/".format(i))
                    for i in range(2, int(end_apge/2))]
                for page_url in page_urls:
                    data = {"cityname": cityname, "city_code": city_code, "url": page_url, "host": hd[‘Host‘], "page": False}
                    yield scrapy.Request(page_url, headers=hd, callback=self.parse_page, meta=data)

            except Exception as e:

                print("没有分页: ", e)


    def parse_item(self, response):
        #
        fname = ‘%s_%s_%s_%s.html‘ % (socket.gethostname(),
                                      response.url.split(‘//‘)[-1].split(‘/‘)[0].replace(‘.‘, ‘-‘),
                                      hashlib.md5(response.url.encode()).hexdigest(),
                                      str(time.time()).split(‘.‘)[0])
        doc = response.text + ‘\n<!--%s-->‘ % response.url
        return {‘path‘: self.path_dir + fname, ‘data‘: doc}