cvpr论文爬取——热词云展示（WEB）

 1 import re
 2 import requests
 3 import pymysql
 4 
 5 def insertCvpr(value):
 6     db = pymysql.connect("localhost", "root", "root", "cvprlist", charset=‘utf8‘)  # 连接数据库
 7 
 8     cursor = db.cursor()
 9     sql="""insert into cvpr values(%s,%s,%s,%s)"""
10     try:
11         cursor.execute(sql, value)
12         db.commit()
13         print(‘插入数据成功‘)
14     except:
15         db.rollback()
16         print("插入数据失败")
17     db.close()
18 
19 
20 url="http://openaccess.thecvf.com/ICCV2019.py";
21 header={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36 Edg/81.0.416.53"}
22 res=requests.get(url,headers=header);
23 res.encoding="utf-8";
24 list=re.findall("""<dt class="ptitle"><br><a href="(.*?)">.*?</a></dt>""",res.text,re.S);
25 for item in list:
26     # print(item)
27     res=requests.get("http://openaccess.thecvf.com/"+item) #爬取到的网站是相对路径，所以要补全，下方同理
28     res.encoding="utf-8"
29     title=re.findall("""<div id="papertitle">(.*?)</div>""",res.text,re.S)
30     summry=re.findall("""<div id="abstract" >(.*?)</div>""",res.text,re.S)
31     link=re.findall("""\[<a href="\.\./\.\./(.*?)">pdf</a>\]""",res.text,re.S)
32     if(len(title)>0):   #有的网站可能爬取不到，数组为空，直接获取会导致程序崩溃
33         insertCvpr((title[0].replace("\n", ""),summry[0].replace("\n", ""),title[0].replace("\n", ""),"http://openaccess.thecvf.com/"+link[0]))

 1 <script type="text/javascript" src="js/echarts.js"></script>
 2 <script type="text/javascript" src="layui/layui.all.js"></script>
 3 <script type="text/javascript" src="js/echarts-cloud.js"></script>
 4 <script type="text/javascript">
 5 option = {
 6         title: {
 7             text: ‘词云‘,//标题
 8             x: ‘center‘,
 9             textStyle: {
10                 fontSize: 23
11             }
12      
13         },
14         backgroundColor: ‘#F7F7F7‘,
15         tooltip: {
16             show: true
17         },
18         series: [{
19             name: ‘热词分析‘,//数据提示窗标题
20             type: ‘wordCloud‘,
21             sizeRange: [6, 66],//画布范围，如果设置太大会出现少词（溢出屏幕）
22             rotationRange: [-45, 90],//数据翻转范围
23             shape: ‘circle‘,
24             textPadding: 0,
25             autoSize: {
26                 enable: true,
27                 minSize: 6
28             },
29             textStyle: {
30                 normal: {
31                     color: function() {
32                         return ‘rgb(‘ + [
33                             Math.round(Math.random() * 160),
34                             Math.round(Math.random() * 160),
35                             Math.round(Math.random() * 160)
36                         ].join(‘,‘) + ‘)‘;
37                     }
38                 },
39                 emphasis: {
40                     shadowBlur: 10,
41                     shadowColor: ‘#333‘
42                 }
43             },
44             
45         }]
46     };
47 
48 $(function(){
49     $.post(
50         ‘getCvprList‘,   //初始化获取所有的文章
51         function(msg){
52             showCvpr(msg);
53         }
54     )
55     $.post(
56         ‘getKeyWord‘,     //获取热词形成热词云
57         function(msg){
58             var json=JSON.parse(msg);
59             var size=json.length;
60             data=[];
61             for(i=0;i<size;i++)
62                 data.push({
63                     name:json[i].key,
64                     value:json[i].num
65                 })
66
67             var charts=echarts.init(document.getElementById(‘cloud‘));
68             option.series[0].data=data;
69             charts.setOption(option);
70                charts.on(‘click‘, function (params) {   //点击传回热词名称，并进行查询展示
71                    $(‘#container‘).html("");
72                    var name = params.data.name;
73                    console.log(name);
74                    $.post(
75                             ‘getCvprList‘,
76                             {‘key‘:name},
77                             function(msg){
78                                 showCvpr(msg);
79                             }
80                         )
81                         
82                 });
83         }
84     )
85 })

 1 function showCvpr(msg){
 2     $(‘#container‘).html("");   //每次展示清空容器
 3     var json=JSON.parse(msg);
 4     var size=json.length;
 5     for(i=0;i<size;i++)
 6         $(‘#container‘).append("<div class=‘layui-colla-item‘ lay-accordion>"+
 7                 "<h2 class=‘layui-colla-title‘><a target=‘_blank‘></a></h2>"+
 8                 "<div class=‘layui-colla-content layui-show‘></div>"+
 9                 "</div>");
10     $(‘.layui-colla-item‘).each(function(i){
11         console.log(json[i].summry);
12         $(this).find(‘.layui-colla-title>a‘).attr(‘href‘,json[i].link).text(json[i].title);
13         $(this).find(‘.layui-colla-content‘).text(json[i].summry);
14     })
15 }