使用flask和BeautifulSoup展示关注的雪球组合信息

  1 # -*- coding: utf-8 -*-
  2 
  3 import json
  4 import re
  5 import urllib.request
  6 from bs4 import BeautifulSoup
  7 from flask import Flask, render_template, request, jsonify
  8 import time
  9 import pandas
 10 
 11 app = Flask(__name__)
 12 app.config[‘JSON_AS_ASCII‘] = False
 13 projects = {}
 14 ZHs0={}
 15 ZHs1={}
 16 cookie = ‘s=7017rril9u; xq_a_token=c4a084fe79a31a6ead299d4d49d622cab3b3b65e; xqat=c4a084fe79a31a6ead299d4d49d622cab3b3b65e; xq_r_token=fc287dc024ce197b1a0e1def6f674c260357bebf; xq_is_login=1; u=1180102135; xq_token_expire=Tue%20Mar%2014%202017%2016%3A56%3A10%20GMT%2B0800%20(CST); bid=45efaa8643ba70c7f4357d0930ff99d4_iz9kzepe‘
 17 
 18 def prof(url_ap0):
 19     url = ‘https://xueqiu.com/cubes/rebalancing/history.json?cube_symbol=‘+url_ap0+‘&count=20&page=1‘
 20     req = urllib.request.Request(url,headers = {
 21            ‘User-Agent‘: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36‘,
 22            ‘cookie‘:cookie
 23            })
 24     html = urllib.request.urlopen(req).read().decode(‘utf-8‘)
 25     data = json.loads(html)
 26 
 27     for i in range (len(data[‘list‘])):
 28         for j in range(len(data[‘list‘][i][‘rebalancing_histories‘])):
 29             if pandas.isnull(data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘]):
 30                 data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘] = str(0)
 31             else:
 32                 data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘] = str(data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘])
 33             if pandas.isnull(data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘]):
 34                 data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘] = str(0)
 35             else:
 36                 data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘] = str(data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘])
 37     try:
 38         for i in range(len(data[‘list‘])):
 39             localtime = time.strftime("%y-%m-%d %H:%M:%S", time.localtime(data[‘list‘][i][‘updated_at‘] / 1000))
 40             if (time.time() - (data[‘list‘][i][‘updated_at‘] / 1000)) < 86400*20:
 41                 for j in range(len(data[‘list‘][i][‘rebalancing_histories‘])):
 42                     ZHs1[j-data[‘list‘][i][‘updated_at‘]]=(localtime,url_ap0,ZHs0[url_ap0],data[‘list‘][i][‘rebalancing_histories‘][j][‘stock_name‘] + ‘: ‘+ data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘] + ‘% ‘ + ("?" if float(data[‘list‘][i][‘rebalancing_histories‘][j][‘prev_weight_adjusted‘])<float(data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘]) else "?") + ‘ ‘+ data[‘list‘][i][‘rebalancing_histories‘][j][‘target_weight‘] + ‘%‘)
 43     except:
 44         print("exception occured")
 45 
 46 
 47 def get_xueqiu_hold(url):
 48     req = urllib.request.Request(url,headers = {
 49                 ‘User-Agent‘: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36‘,
 50                 ‘cookie‘:cookie
 51                })
 52     soup = urllib.request.urlopen(req).read().decode(‘utf-8‘)
 53     soup = BeautifulSoup(soup, ‘lxml‘)
 54     script = soup.find(‘script‘, text=re.compile(‘SNB\.cubeInfo‘))
 55     json_text = re.search(r‘^\s*SNB\.cubeInfo\s*=\s*({.*?})\s*;\s*$‘,
 56                       script.string, flags=re.DOTALL | re.MULTILINE).group(1)
 57     data = json.loads(json_text)
 58     for d in data["view_rebalancing"]["holdings"]:
 59         if d[‘stock_name‘] in projects.keys():
 60             projects[d[‘stock_name‘]] += d[‘weight‘]            
 61         else:
 62             projects[d[‘stock_name‘]]= d[‘weight‘]
 63     
 64 
 65 @app.route("/", methods=[‘GET‘, ‘POST‘])
 66 def index():
 67     return render_template("index.html")
 68 
 69 
 70 @app.route(‘/start‘, methods=[‘POST‘])
 71 def post_url():
 72     # get url
 73     projects.clear()
 74     ZHs0.clear()
 75     ZHs1.clear()    
 76     data = json.loads(request.data.decode())
 77     url = data["url"]
 78     url0 = ‘https://xueqiu.com/stock/portfolio/stocks.json?size=1000&pid=-1&tuid=‘+url+‘&cuid=1180102135&_=1477728185503‘
 79     req = urllib.request.Request(url0,headers = {
 80            ‘User-Agent‘: ‘Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36‘,
 81            ‘cookie‘:cookie
 82            })
 83     html = urllib.request.urlopen(req).read().decode(‘utf-8‘)
 84     data = json.loads(html)
 85     for item in data["stocks"]:
 86         if re.search(‘ZH\d{6}‘,str(item)):
 87             ZHs0[item["code"]]=item["stockName"]
 88     for ZH0 in ZHs0:
 89         prof(ZH0)
 90     ZHs = re.findall(‘ZH\d{6}‘,data["portfolios"][0]["stocks"])
 91     for ZH in ZHs:
 92         get_xueqiu_hold("https://xueqiu.com/P/"+ZH)
 93     return url
 94 
 95 @app.route("/data")
 96 def data():
 97     projects0 = sorted(projects.items(), key=lambda x: (-x[1],x[0]))[:12]
 98     return jsonify(dict(projects0))
 99 
100 @app.route("/trans0")
101 def trans0():
102     return jsonify(ZHs0)
103 
104 @app.route("/trans1")
105 def trans1():
106     return jsonify(ZHs1)
107 
108 if __name__ == "__main__":
109     app.run(host=‘0.0.0.0‘,port=5000,debug=True)

 1 <!DOCTYPE html>
 2 <html ng-app="XueqiuholdApp">
 3   <head>
 4     <link rel="shortcut icon" href="https://assets.imedao.com/images/vipicon_4@2x.png" />
 5     <title>你关注的雪球组合持仓</title>
 6     <meta name="viewport" content="width=device-width, initial-scale=1.0">
 7     <!-- styles -->
 8 <!-- 新 Bootstrap 核心 CSS 文件 -->
 9 <link href="http://cdn.static.runoob.com/libs/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet">
10 
11 <!-- 可选的Bootstrap主题文件（一般不使用） -->
12 <script src="http://cdn.static.runoob.com/libs/bootstrap/3.3.7/css/bootstrap-theme.min.css"></script>
13 <link rel="stylesheet" type="text/css" href="{{ url_for(‘static‘, filename=‘main.css‘) }}">
14 
15 
16   </head>
17     <body ng-controller="XueqiuholdController">
18     <div class="container">
19       <div class="row">
20         <div class="col-sm-7">
21           <div class="row-sm-7 col-sm-offset-1">
22           <h2>你的雪球ID</h2>
23           <br>
24           <form role="form" ng-submit="getResults()">
25             <div class="form-group">
26               <input type="text" name="url" class="form-control" id="url-box" placeholder="雪球ID" style="max-width: 300px;" ng-model="url" required>
27             </div>
28             {% raw %}
29               <button type="submit" class="btn btn-primary" ng-disabled="loading">{{ submitButtonText }}</button>
30             {% endraw %}
31           </form>
32           <div class="alert alert-danger" role="alert" ng-show=‘urlError‘>
33             <span aria-hidden="true"></span>
34             <span class="sr-only">Error:</span>
35             <span>There was an error submitting your URL.<br>
36             Please check to make sure it is valid before trying again.</span>
37           </div> 
38         </div>
39           {% raw %}
40           <br>
41           <h4>关注组合最新调仓：</h4>
42           <div style="overflow: auto;">
43           <table class="table table-fixed table-striped">
44           <tbody>
45             <tr class="table-row" ng-repeat="(key, val) in trans">
46               <td Style="text-align:middle" item-width="162">{{val[0]}}</td>
47               <td Style="text-align:left" item-width="132"> {{val[2]}} </td>
48               <td Style="text-align:left" item-width="132"> {{val[1]}} </td>
49               <td Style="text-align:right" item-width="256"> {{val[3]}}</td>
50             </tr>
51           </tbody>
52           </table> 
53           </div>
54           {% endraw %}
55         </div>
56         <div class="col-sm-3 col-sm-offset-1">
57           <br><br>
58           <h4>关注组合累计仓位</h4>
59           <div id="results">
60             <table class="table table-striped">
61               <thead>
62                 <tr>
63                   <th>Stock</th>
64                   <th>Percent</th>
65                 </tr>
66               </thead>
67               {% raw %}
68               <tbody>
69                 <tr ng-repeat="(key, val) in xueqiuhold | orderBy:‘key‘">
70                   <td>{{key}}</td>
71                   <td>{{val.toFixed(2)}}%</td>
72                 </tr>
73               </tbody>
74               {% endraw %}
75             </table>
76           </div>
77           <img class="col-sm-3 col-sm-offset-4" src="{{ url_for(‘static‘,
78             filename=‘spinner.gif‘) }}" ng-show="loading">
79         </div>
80       </div>
81       <br>
82       <div class="row-sm-12 row-sm-offset-1">      
83       <wordcount-chart data="xueqiuhold"></wordcount-chart>
84       </div>
85     </div>
86     <br><br>
87     <!-- scripts -->
88     <script src="{{ url_for(‘static‘, filename=‘d3.js‘) }}" charset="utf-8"></script>
89     <script src="{{ url_for(‘static‘, filename=‘angular.js‘) }}"></script>
90     <script src="{{ url_for(‘static‘, filename=‘main.js‘) }}"></script>
91     <!-- jQuery文件。务必在bootstrap.min.js 之前引入 -->
92 <script src="http://cdn.static.runoob.com/libs/jquery/2.1.1/jquery.min.js"></script>
93 
94 <!-- 最新的 Bootstrap 核心 JavaScript 文件 -->
95 <script src="http://cdn.static.runoob.com/libs/bootstrap/3.3.7/js/bootstrap.min.js"></script>
96   </body>
97 </html>

  1 (function () {
  2 
  3   ‘use strict‘;
  4 
  5   angular.module(‘XueqiuholdApp‘, [])
  6 
  7   .controller(‘XueqiuholdController‘, [‘$scope‘, ‘$log‘, ‘$http‘, ‘$timeout‘,
  8     function($scope, $log, $http, $timeout) {
  9 
 10     $scope.submitButtonText = ‘Submit‘;
 11     $scope.loading = false;
 12     $scope.urlerror = false;
 13 
 14     $scope.getResults = function() {
 15 
 16       $log.log(‘test‘);
 17 
 18       // get the URL from the input
 19       var userInput = $scope.url;
 20 
 21       // fire the API request
 22       $http.post(‘/start‘, {‘url‘: userInput}).
 23         success(function(results) {
 24           $log.log(results);
 25           getXueqiuHold();
 26           $scope.xueqiuhold = null;
 27           $scope.trans = null;
 28           $scope.loading = true;
 29           $scope.submitButtonText = ‘Loading...‘;
 30           $scope.urlerror = false;
 31         }).
 32         error(function(error) {
 33           $log.log(error);
 34         });
 35 
 36     };
 37 
 38     function getXueqiuHold() {
 39 
 40       var timeout = ‘‘;
 41 
 42       var poller = function() {
 43         // fire another request
 44         $http.get(‘/data‘).
 45           success(function(data, status, headers, config) {
 46             if(status === 202) {
 47               $log.log(data, status);
 48             } else if (status === 200){
 49               $log.log(data);
 50               $scope.loading = false;
 51               $scope.submitButtonText = "Submit";
 52               $scope.xueqiuhold = data;
 53                 $http.get(‘/trans1‘).
 54                   success(function(data, status, headers, config) {
 55                     if(status === 202) {
 56                       $log.log(data, status);
 57                     } else if (status === 200){
 58                       $log.log(data);
 59                       $scope.trans = data;
 60               }});
 61               $timeout.cancel(timeout);
 62               return false;
 63             }
 64             // continue to call the poller() function every 2 seconds
 65             // until the timeout is cancelled
 66             timeout = $timeout(poller, 2000);
 67           }).
 68           error(function(error) {
 69             $log.log(error);
 70             $scope.loading = false;
 71             $scope.submitButtonText = "Submit";
 72             $scope.urlerror = true;
 73           });
 74       };
 75 
 76       poller();
 77 
 78     }
 79 
 80   }])
 81 
 82   .directive(‘wordcountChart‘, [‘$parse‘, function ($parse) {
 83     return {
 84       restrict: ‘E‘,
 85       replace: true,
 86       template: ‘<div id="chart"></div>‘,
 87       link: function (scope) {
 88         scope.$watch(‘xueqiuhold‘, function() {
 89           d3.select(‘#chart‘).selectAll(‘*‘).remove();
 90           var data = scope.xueqiuhold;
 91           for (var word in data) {
 92             d3.select(‘#chart‘)
 93               .append(‘div‘)
 94               .selectAll(‘div‘)
 95               .data(word[0])
 96               .enter()
 97               .append(‘div‘)
 98               .style(‘width‘, function() {
 99                 return (data[word]*3) + ‘px‘;
100               })
101               .text(function(d){
102                 return word;
103               });
104           }
105         }, true);
106       }
107      };
108   }]);
109 
110 }());