基于朴素贝叶斯分类算法的邮件过滤系统

 1 #=============================================
 2 #    输入:
 3 #        bigString:       待转换文档字符串
 4 #    输出:
 5 #        待转换文档的列表格式
 6 #=============================================
 7 def textParse(bigString):
 8     import re
 9     listOfTokens = re.split(r‘\W*‘, bigString)
10     return [tok.lower() for tok in listOfTokens if len(tok) > 2]

 1 #=============================================
 2 #    输入:
 3 #        vocabList:     词汇列表
 4 #        inputSet:       待转换文档的列表格式
 5 #    输出:
 6 #        returnVec:    转换后的词向量(词袋模型)
 7 #============================================= 
 8 def bagOfWords2VecMN(vocabList, inputSet):
 9     ‘文档(列表格式) -> 词向量(词袋模型)‘
10     
11     returnVec = [0]*len(vocabList)
12     for word in inputSet:
13         if word in vocabList:
14             returnVec[vocabList.index(word)] += 1
15             
16     return returnVec
17 
18 #=============================================
19 #    输入:
20 #        代码内指定路径的两种分类邮件
21 #    输出:
22 #        空 (打印贝叶斯分类测试的结果)
23 #=============================================    
24 def spamTest():
25     ‘测试贝叶斯分类并打印结果‘
26     
27     # 文档字符串集合
28     docList=[]
29     # 文档分类集合
30     classList = []
31     # 所有字符串
32     fullText =[]
33     
34     # 从两种分类邮件中各取出25封邮件，获得文档字符串集合，文档分类集合，所有字符串。
35     for i in range(1,26):
36         wordList = textParse(open(‘/home/fangmeng/email/spam/%d.txt‘ % i).read())
37         docList.append(wordList)
38         fullText.extend(wordList)
39         classList.append(1)
40         wordList = textParse(open(‘/home/fangmeng/email/ham/%d.txt‘ % i).read())
41         docList.append(wordList)
42         fullText.extend(wordList)
43         classList.append(0)
44      
45     # 词汇表
46     vocabList = createVocabList(docList)
47     # 训练集范围
48     trainingSet = range(50)
49     # 测试集范围
50     testSet=[]
51     
52     # 这总共50封邮件里，取10封用来做分类测试。
53     # 同时将这10封测试邮件从训练集范围内移除。
54     for i in range(10):
55         randIndex = int(random.uniform(0,len(trainingSet)))
56         testSet.append(trainingSet[randIndex])
57         del(trainingSet[randIndex])
58     
59     # 训练集词向量矩阵    
60     trainMat=[]
61     # 训练集分类列表
62     trainClasses = []
63     
64     # 构建训练集词向量矩阵和训练集分类列表
65     for docIndex in trainingSet:
66         trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
67         trainClasses.append(classList[docIndex])
68     
69     # 对训练集中邮件进行分类测试并打印测试结果    
70     p0V,p1V,pSpam = trainNB0(numpy.array(trainMat),numpy.array(trainClasses))
71     errorCount = 0
72     for docIndex in testSet:
73         wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
74         if classifyNB(numpy.array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
75             errorCount += 1
76             print "错误分类:\n",docList[docIndex]
77     print ‘错误率: \n‘,float(errorCount)/len(testSet)