Python简单实现基于VSM的余弦相似度计算

  1 # -*- coding: utf-8 -*-
  2 import time          
  3 import re          
  4 import os
  5 import string
  6 import sys
  7 import math
  8 
  9 
 10 ‘‘‘ ------------------------------------------------------- ‘‘‘
 11 #统计关键词及个数
 12 def CountKey(fileName, resultName):
 13     try:
 14         #计算文件行数
 15         lineNums = len(open(fileName,‘rU‘).readlines())
 16         print u‘文件行数: ‘ + str(lineNums)
 17 
 18         #统计格式 格式<Key:Value> <属性:出现个数>
 19         i = 0
 20         table = {}
 21         source = open(fileName,"r")
 22         result = open(resultName,"w")
 23         
 24         while i < lineNums:
 25             line = source.readline()
 26             line = line.rstrip(‘\n‘)
 27             print line
 28             
 29             words = line.split(" ")  #空格分隔
 30             print str(words).decode(‘string_escape‘) #list显示中文
 31             
 32             #字典插入与赋值
 33             for word in words:
 34                 if word!="" and table.has_key(word):      #如果存在次数加1
 35                     num = table[word]
 36                     table[word] = num + 1
 37                 elif word!="":                            #否则初值为1
 38                     table[word] = 1
 39             i = i + 1
 40 
 41         #键值从大到小排序 函数原型：sorted(dic,value,reverse)
 42         dic = sorted(table.iteritems(), key = lambda asd:asd[1], reverse = True)
 43         for i in range(len(dic)):
 44             #print ‘key=%s, value=%s‘ % (dic[i][0],dic[i][1])
 45             result.write("<"+dic[i][0]+":"+str(dic[i][1])+">\n")
 46         return dic
 47         
 48     except Exception,e:    
 49         print ‘Error:‘,e
 50     finally:
 51         source.close()
 52         result.close()
 53         print ‘END\n\n‘
 54 
 55 
 56 ‘‘‘ ------------------------------------------------------- ‘‘‘
 57 #统计关键词及个数 并计算相似度
 58 def MergeKeys(dic1,dic2):
 59     #合并关键词 采用三个数组实现
 60     arrayKey = []
 61     for i in range(len(dic1)):
 62         arrayKey.append(dic1[i][0])       #向数组中添加元素
 63     for i in range(len(dic2)):       
 64         if dic2[i][0] in arrayKey:
 65             print ‘has_key‘,dic2[i][0]
 66         else:                             #合并
 67             arrayKey.append(dic2[i][0])
 68     else:
 69         print ‘\n\n‘
 70     
 71     test = str(arrayKey).decode(‘string_escape‘)  #字符转换
 72     print test
 73 
 74     #计算词频 infobox可忽略TF-IDF
 75     arrayNum1 = [0]*len(arrayKey)
 76     arrayNum2 = [0]*len(arrayKey)
 77     
 78     #赋值arrayNum1
 79     for i in range(len(dic1)):     
 80         key = dic1[i][0]
 81         value = dic1[i][1]
 82         j = 0
 83         while j < len(arrayKey):
 84             if key == arrayKey[j]:
 85                 arrayNum1[j] = value
 86                 break
 87             else:
 88                 j = j + 1
 89 
 90     #赋值arrayNum2
 91     for i in range(len(dic2)):     
 92         key = dic2[i][0]
 93         value = dic2[i][1]
 94         j = 0
 95         while j < len(arrayKey):
 96             if key == arrayKey[j]:
 97                 arrayNum2[j] = value
 98                 break
 99             else:
100                 j = j + 1
101     
102     print arrayNum1
103     print arrayNum2
104     print len(arrayNum1),len(arrayNum2),len(arrayKey)
105 
106     #计算两个向量的点积
107     x = 0
108     i = 0
109     while i < len(arrayKey):
110         x = x + arrayNum1[i] * arrayNum2[i]
111         i = i + 1
112     print x
113 
114     #计算两个向量的模
115     i = 0
116     sq1 = 0
117     while i < len(arrayKey):
118         sq1 = sq1 + arrayNum1[i] * arrayNum1[i]   #pow(a,2)
119         i = i + 1
120     print sq1
121     
122     i = 0
123     sq2 = 0
124     while i < len(arrayKey):
125         sq2 = sq2 + arrayNum2[i] * arrayNum2[i]
126         i = i + 1
127     print sq2
128     
129     result = float(x) / ( math.sqrt(sq1) * math.sqrt(sq2) )
130     return result
131     
132 
133 ‘‘‘ ------------------------------------------------------- 
134     基本步骤：
135         1.分别统计两个文档的关键词
136         2.两篇文章的关键词合并成一个集合,相同的合并,不同的添加
137         3.计算每篇文章对于这个集合的词的词频 TF-IDF算法计算权重
138         4.生成两篇文章各自的词频向量
139         5.计算两个向量的余弦相似度,值越大表示越相似                             
140     ------------------------------------------------------- ‘‘‘
141 #主函数
142 def main():
143     #计算文档1-百度的关键词及个数
144     fileName1 = "BaiduSpider.txt"
145     resultName1 = "Result_Key_BD.txt"
146     dic1 = CountKey(fileName1, resultName1)
147     
148     #计算文档2-互动的关键词及个数
149     fileName2 = "HudongSpider\\001.txt"
150     resultName2 = "HudongSpider\\Result_Key_001.txt"
151     dic2 = CountKey(fileName2, resultName2)
152 
153     #合并两篇文章的关键词及相似度计算
154     result = MergeKeys(dic1, dic2)
155     print result
156 
157 
158 if __name__ == ‘__main__‘:
159     main()