通过爬虫抓取链家二手房数据

# 二手房微信小程序接口
https://wechat.lianjia.com/ershoufang/search?city_id=120000&condition=&query=&order=&offset=0&limit=10&sign=
#headers 参数
lianjia-source:ljwxapp
authorization:bGp3eGFwcDoxYTNjMDA3MmQ0ZDA3NTM2ODVlOTJlMDQ0NmUwNDk5NQ==
time-stamp:1527659945696

C:\Users\hunk>adb --version
Android Debug Bridge version 1.0.39
Version 0.0.1-4500957
Installed as D:\Program Files\android-sdk-windows\platform-tools\adb.exe

#链接mumu 模拟器
adb connect 127.0.0.1:7555   # 注意adb连接手机端口5555
# 
adb shell
cd /data/data/com.tencent.mm/MicroMsg/8c12ff3e9b0391b589c0d5b16dc21952/appbrand/pkg   #8c12ff3e9b0391b589c0d5b16dc21952为当前微信的用户名字，可以根据自己的实际来查找
cancro:/data/data/com.tencent.mm/MicroMsg/8c12ff3e9b0391b589c0d5b16dc21952/appbrand/pkg # rm -rf *   # 便于识别那个属于链家，先删除目录内容再次打开链家小程序
cancro:/data/data/com.tencent.mm/MicroMsg/8c12ff3e9b0391b589c0d5b16dc21952/appbrand/pkg # ls
_-1261323258_17.wxapkg   # 名字不固定，根据实际来查看
_1123949441_130.wxapkg
and/pkg # cp _-1261323258_17.wxapkg  /sdcard/
cancro:/data/data/com.tencent.mm/MicroMsg/8c12ff3e9b0391b589c0d5b16dc21952/appbrand/pkg # exit
# 把_-1261323258_17.wxapkg 下载本地

C:\Users\hunk>adb pull /sdcard/_-1261323258_17.wxapkg . /sdcard/_-1261323258_17.wxapkg: 1 file pulled. 1.0 MB/s (988967 bytes in 0.941s)

 python .\unwxapkg.py .\_-1261323258_17.wxapkg

/*定义变量 l，空字符串*/
var l = "";
/*   
1.将请求参数解析成key:value 键值对; 
2.通过key进行从小到大排序;
3.通过"="将 key和value连接起来，并追加到变量 l中;
 */
Object.keys(a.data).sort().forEach(function (e) {
    return l += e + "=" + ("object" == t(a.data[e]) ? JSON.stringify(a.data[e]) : a.data[e])
}),
/*l 中再次追加字符串6e8566e348447383e16fdd1b233dbb49 */
l += "6e8566e348447383e16fdd1b233dbb49",
/*对l 字符串进行md5加密*/
l = e.default.hexMD5(l),
/*对加密的结果添加前缀 ljwxapp:*/
l = "ljwxapp:" + l,
c.Authorization = n.encode(l),

import hashlib
import base64
from urllib.parse import urlparse, parse_qs

app_id = "ljwxapp:"
app_key = "6e8566e348447383e16fdd1b233dbb49"


def get_authorization(url):
    """
    根据url 动态获取authorization
    :param url:
    :return:
    """

    param = ""
    parse_param = parse_qs(urlparse(url).query, keep_blank_values=True)  # 解析url参数
    data = {key: value[-1] for key, value in parse_param.items()}  # 生成字典
    dict_keys = sorted(data.keys())  # 对key进行排序
    for key in dict_keys:  # 排序后拼接参数,key = value 模式
        param += str(key) + "=" + data[key]
    param = param + app_key  # 参数末尾添加app_key
    param_md5 = hashlib.md5(param.encode()).hexdigest()  # 对参数进行md5 加密
    authorization_source = app_id + param_md5  # 加密结果添加前缀app_id
    authorization = base64.b64encode(authorization_source.encode())  # 再次进行base64 编码
    return authorization.decode()


if __name__ in "__main__":
　　# 天津链家二手房信息
    url = "https://wechat.lianjia.com/ershoufang/search?city_id=120000&condition=&query=&order=&offset=0&limit=10&sign="
    print(get_authorization(url))  # 生成加密串：bGp3eGFwcDoxYTNjMDA3MmQ0ZDA3NTM2ODVlOTJlMDQ0NmUwNDk5NQ==

# -*- coding: utf-8 -*-
# @Time    : 2018/5/30 10:52
# @Author  : Hunk
# @File    : ErShouFangListItems.py
# @Software: PyCharm

from scrapy import Item
from scrapy import Field


class ErShouFangItems(Item):
    house_code = Field()  # 二手房ID
    resblock_id = Field()  # 小区ID
    resblock_name = Field()  # 小区名字
    price = Field()  # 价格元/平

# -*- coding: utf-8 -*-
# @Time    : 2018/5/30 10:51
# @Author  : Hunk
# @File    : ParseErShouFangList.py
# @Software: PyCharm
import time
import json
import math
from scrapy import Spider
from scrapy import Request
from lib.Encrypt import get_authorization
from scrapy.utils.project import get_project_settings
from ..items.ErShouFangListItems import ErShouFangItems


class ParseErShouFangSpider(Spider):
    """
    楼盘列表
    """
    name = ‘TJErShouFang‘

    def __init__(self, city_id, *args, **kwargs):
        super(ParseErShouFangSpider, self).__init__(*args, **kwargs)
        self.settings = get_project_settings()
        self.base_url = self.settings["WX_BASE_URL"]  # 获取配置文件中的微信小程序的请求地址
        self.api = "/ershoufang/search"  # 二手房接口地址
        self.city_id = city_id
        self.limit_offset = 0
        self.new_house_url = self.base_url + self.api + "?city_id=%s&condition=&query=&order=&offset=%s&limit=10&sign"  # 拼接二手房接口地址
        self.start_urls = self.new_house_url % (city_id, self.limit_offset)  # 请求地址
        self.headers = {"time-stamp": str(int(time.time() * 1000)), "lianjia-source": "ljwxapp",
                        "authorization": ""}  # 定义header

    def start_requests(self):
        """
        重写start_requests
        :return:
        """
        self.headers["authorization"] = get_authorization(self.start_urls)
        yield Request(url=self.start_urls, headers=self.headers, callback=self.parse)

    def parse(self, response):

        total_count = self.parse_page(response)  # 获取一共有多少页面数据
        for i in range(0, total_count):  # 翻页操作
            url = self.new_house_url % (self.city_id, self.limit_offset)  # 定义请求的url
            self.limit_offset += 10  # 每页显示10条数据，每次翻页递增10条
            self.headers["authorization"] = get_authorization(url)
            yield Request(url=url, headers=self.headers, callback=self.parse_house_item)

    def parse_house_item(self, response):
        """
        解析JSON 数据
        :param response:
        :return:
        """
        item = ErShouFangItems()
        content = json.loads(response.body.decode())
        ershoufang_list = content["data"]["list"]
        if len(ershoufang_list) > 0:
            for ershoufang in ershoufang_list:
                item["house_code"] = ershoufang_list[ershoufang]["house_code"]
                item["resblock_id"] = ershoufang_list[ershoufang]["resblock_id"]
                item["resblock_name"] = ershoufang_list[ershoufang]["resblock_name"]
                yield item

    def parse_page(self, response):
        """
        计算总共页数
        :param response:
        :return:
        """
        content = json.loads(response.body.decode())
        total_count = int(content["data"]["total_count"])
        return int(math.ceil(total_count / 10))