码迷,mamicode.com
首页 > 编程语言 > 详细

python爬虫算一下demo大师网站的总创收

时间:2018-03-26 16:52:37      阅读:203      评论:0      收藏:0      [点我收藏+]

标签:爬虫




    用python爬虫算一下demo大师网站的总创收。。。。。。

技术分享图片

技术分享图片

#!/usr/bin/env python
#coding:utf-8

import requests
import json
from bs4 import BeautifulSoup

def demodashi(url):
    response = requests.get(url)
    html = response.text
    html = json.loads(html)
    totalPage = html['totalPage']
    currentPage = html['currentPage']
    data = html['data']
    demouserdict = {}
    for demo in data:
        demouserdict[demo['ID']] = demo
    pircesumpage = 0
    for id in demouserdict:
        downnum = demouserdict[id]['DOWN_NUM']
        usrurl = 'http://www.demodashi.com/demo/%s.html' % id
        header={
            'Accept': 'text / html, application / xhtml + xml, application / xml;q = 0.9, image / webp, * / *;q = 0.8',
            'Accept - Language': 'zh - CN, zh;q = 0.8',
            'Cookie': 'userIdInfoCookie = XYq4JJPg8Pgw8LEUMhD;userSessionInfoCookie = 1ABE726154AB6AA9E8E771AD3188ED8C;JSESSIONID = BBAA6DCFC9E2CE2568AEEBF38EB3F101;Hm_lvt_27ece87947b753bb8e85404fdffce737 = 1521615577, 1521716172, 1522033433, 1522042544;Hm_lpvt_27ece87947b753bb8e85404fdffce737 = 1522042544',
            'User - Agent': 'Mozilla / 5.0(Windows NT 6.1;WOW64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 55.0.2883.87UBrowser / 6.2.3964 .2Safari / 537.36',
        }
        response = requests.get(usrurl,headers=header)
        html = response.text
        soup = BeautifulSoup(html,'lxml')
        try:
            pirce = soup.find('div',class_='sj-dm dsj-l').b.string
        except:
            pirce = 0
        pircesumpage += int(pirce)*int(downnum)
    return pircesumpage,totalPage

def bankuai(url):
    totalPage = demodashi(url)[1]
    totalpirce = 0
    for page in range(0,totalPage):
        url = 'http://www.demoku.com/portal/demo/getDemoList4page?queryString=&toPage=%s&orderBy=&type=&typeId=&ptype=&to1=demo&to2=buy&userId=' % page
        totalpirce += demodashi(url)[0]
    return totalpirce

if __name__ == '__main__':
    '''
    全部 :http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=&type=&typeId=&ptype=&to1=demo&to2=buy&userId=

    全部 (热门、推荐、最新)
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=&typeId=&ptype=&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=&typeId=&ptype=&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=&typeId=&ptype=&to1=demo&to2=buy&userId=

    大数据
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=2&ptype=2&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=2&ptype=2&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=2&ptype=2&to1=demo&to2=buy&userId=

    移动技术
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=9&ptype=9&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=9&ptype=9&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=9&ptype=9&to1=demo&to2=buy&userId=

    web系列
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=12&ptype=12&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=12&ptype=12&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=12&ptype=12&to1=demo&to2=buy&userId=


    前端开发
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=26&ptype=26&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=26&ptype=26&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=26&ptype=26&to1=demo&to2=buy&userId=


    系统与编程
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=31&ptype=31&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=31&ptype=31&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=31&ptype=31&to1=demo&to2=buy&userId=


    互联网应用
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=47&ptype=47&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=47&ptype=47&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=47&ptype=47&to1=demo&to2=buy&userId=

    方案设计
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=50&ptype=50&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=50&ptype=50&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=50&ptype=50&to1=demo&to2=buy&userId=

    产品设计
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=52&ptype=52&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=52&ptype=52&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=52&ptype=52&to1=demo&to2=buy&userId=

    UI设计
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=downLoadMax&type=1&typeId=54&ptype=54&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=avgMarkMax&type=1&typeId=54&ptype=54&to1=demo&to2=buy&userId=
    http://www.demodashi.com/portal/demo/getDemoList4page?queryString=&toPage=0&orderBy=uploadNew&type=1&typeId=54&ptype=54&to1=demo&to2=buy&userId=

    '''
    urllist = ['http://www.demoku.com/portal/demo/getDemoList4page?queryString=&toPage=1&orderBy=&type=&typeId=&ptype=&to1=demo&to2=buy&userId=']
    for url in urllist:
        print 'demo大师网站总收入: ' + str(bankuai(url))


    内容如果侵犯隐私,联系我会马上删除。。。。。  (961769710)

python爬虫算一下demo大师网站的总创收

标签:爬虫

原文地址:http://blog.51cto.com/9399369/2091177

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!