TFIDF代码实现

 1 #!/usr/bin/env python 
 2 # encoding: utf-8 
 3 
 4 """
 5 @author: zkjiang
 6 @site: https://www.github.com
 7 @software: PyCharm
 8 @file: TFIDF.py
 9 @time: 2019/2/2 12:33
10 """
11 
12 import numpy as np
13 
14 class TFIDF(object):
15 
16     """
17     手写一个TFIDF统计类,只写最简单的一个实现
18     """
19 
20     def __init__(self, corpus):
21         """
22         初始化
23         self.vob:词汇个数统计，dict格式
24         self.word_id:词汇编码id，dict格式
25         self.smooth_idf：平滑系数，关于平滑不多解释了
26         :param corpus:输入的语料
27         """
28         self.word_id = {}
29         self.vob = {}
30         self.corpus = corpus
31         self.smooth_idf = 0.01
32 
33     def fit_transform(self, corpus):
34         pass
35 
36     def get_vob_fre(self):
37         """
38         计算文本特特征的出现次数，也就是文本频率term frequency，但是没有除token总数，因为后面bincount计算不支持float
39         :return: 修改self.vob也就是修改词频统计字典
40         """
41         # 统计各词出现个数
42         id = 0
43         for single_corpus in self.corpus:
44             if isinstance(single_corpus, list):
45                 pass
46             if isinstance(single_corpus, str):
47                 single_corpus = single_corpus.strip("\n").split(" ")
48             for word in single_corpus:
49                 if word not in self.vob:
50                     self.vob[word] = 1
51                     self.word_id[word] = id
52                     id += 1
53                 else:
54                     self.vob[word] += 1
55 
56         # 生成矩阵
57         X = np.zeros((len(self.corpus), len(self.vob)))
58         for i in range(len(self.corpus)):
59             if isinstance(self.corpus[i], str):
60                 single_corpus = self.corpus[i].strip("\n").split(" ")
61             else:
62                 single_corpus = self.corpus[i]
63             for j in range(len(single_corpus)):
64                 feature = single_corpus[j]
65                 feature_id = self.word_id[feature]
66                 X[i, feature_id] = self.vob[feature]
67         return X.astype(int)  # 需要转化成int
68 
69 
70     def get_tf_idf(self):
71         """
72         计算idf并生成最后的TFIDF矩阵
73         :return:
74         """
75         X = self.get_vob_fre()
76         n_samples, n_features = X.shape
77         df = []
78         for i in range(n_features):
79             """
80             这里是统计每个特征的非0的数量，也就是逆文档频率指数的分式中的分母，是为了计算idf
81             """
82             df.append(n_samples - np.bincount(X[:,i])[0])
83         df = np.array(df)
84         # perform idf smoothing if required
85         df += int(self.smooth_idf)
86         n_samples += int(self.smooth_idf)
87         idf = np.log(n_samples / df) + 1  # 核心公式
88         # print(self.vob)
89         # print(self.word_id)
90         return X*idf/len(self.vob)
91 
92 
93 
94 if __name__ == ‘__main__‘:
95     corpus = [["我","a","e"],["我","a","c"],["我","a","b"]]
96     test = TFIDF(corpus)
97     # print(test.get_vob_fre())
98     print(test.get_tf_idf())