码迷,mamicode.com
首页 > 编程语言 > 详细

朴素贝叶斯算法的实例

时间:2014-11-19 07:10:16      阅读:371      评论:0      收藏:0      [点我收藏+]

标签:style   blog   http   io   ar   color   os   使用   sp   

贝叶斯的应用

  • 过滤垃圾邮件

    贝叶斯分类器的著名的应用就是垃圾邮件过滤了,这方面推荐想详细了解的可以去看看《黑客与画家》或是《数学之美》中对应的章节,贝叶斯的基础实现看这里

    数据集

    两个文件夹,分别是正常邮件和垃圾邮件,其中各有25封邮件

    测试方法

    从50封邮件中随机选取10封做为测试数据

    实现细节


    1.首先我们需要将文本转成我们需要的向量的样子,这里需要使用一点正则表达式
    2.由于采取交叉验证的方式,随机过程会导致每次的结果不尽相同

 

  1 #coding=utf-8
  2 from numpy import *
  3 
  4 #解析文档的函数
  5 def textParse(bigString):
  6     import re
  7     listOfTokens = re.split(r\W*,bigString)
  8     return [tok.lower() for tok in listOfTokens if len(tok) > 2]
  9     
 10     
 11 #创建一个带有所有单词的列表
 12 def createVocabList(dataSet):
 13     vocabSet = set([])
 14     for document in dataSet:
 15         vocabSet = vocabSet | set(document)
 16     return list(vocabSet)
 17     
 18 def setOfWords2Vec(vocabList, inputSet):
 19     retVocabList = [0] * len(vocabList)
 20     for word in inputSet:
 21         if word in vocabList:
 22             retVocabList[vocabList.index(word)] = 1
 23         else:
 24             print word ,word ,not in dict
 25     return retVocabList
 26 
 27 #另一种模型    
 28 def bagOfWords2VecMN(vocabList, inputSet):
 29     returnVec = [0]*len(vocabList)
 30     for word in inputSet:
 31         if word in vocabList:
 32             returnVec[vocabList.index(word)] += 1
 33     return returnVec
 34 
 35 def trainNB0(trainMatrix,trainCatergory):
 36     numTrainDoc = len(trainMatrix)
 37     numWords = len(trainMatrix[0])
 38     pAbusive = sum(trainCatergory)/float(numTrainDoc)
 39     #防止多个概率的成绩当中的一个为0
 40     p0Num = ones(numWords)
 41     p1Num = ones(numWords)
 42     p0Denom = 2.0
 43     p1Denom = 2.0
 44     for i in range(numTrainDoc):
 45         if trainCatergory[i] == 1:
 46             p1Num +=trainMatrix[i]
 47             p1Denom += sum(trainMatrix[i])
 48         else:
 49             p0Num +=trainMatrix[i]
 50             p0Denom += sum(trainMatrix[i])
 51     p1Vect = log(p1Num/p1Denom)#处于精度的考虑,否则很可能到限归零
 52     p0Vect = log(p0Num/p0Denom)
 53     return p0Vect,p1Vect,pAbusive
 54     
 55 def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
 56     p1 = sum(vec2Classify * p1Vec) + log(pClass1)    #element-wise mult
 57     p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
 58     if p1 > p0:
 59         return 1
 60     else: 
 61         return 0
 62         
 63 def spamTest(spamFloder, hamFloder):
 64     docList = []
 65     classList = []
 66     fullText = []
 67     for i in range(1,26):
 68         wordList = textParse(open(spamFloder+str(i)+.txt).read())
 69         docList.append(wordList)
 70         fullText.extend(wordList)
 71         classList.append(1)
 72         wordList = textParse(open(hamFloder+str(i)+.txt).read())
 73         docList.append(wordList)
 74         fullText.extend(wordList)
 75         classList.append(0)
 76     vocabList = createVocabList(docList)
 77     trainingSet = range(50)
 78     testSet = []
 79     for i in range(10):
 80         randIndex = int(random.uniform(0,len(trainingSet)))
 81         testSet.append(trainingSet[randIndex])
 82         del(trainingSet[randIndex])
 83     trainMat = []
 84     trianClasses = []
 85     print trainingSet
 86     for docIndex in trainingSet:
 87         trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))
 88         #trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
 89         trianClasses.append(classList[docIndex])
 90     p0V,p1V,pSpam = trainNB0(array(trainMat),array(trianClasses))
 91     errorCount = 0
 92     for docIndex in testSet:        #classify the remaining items
 93         #wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
 94         wordVector = setOfWords2Vec(vocabList, docList[docIndex])
 95         if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
 96             errorCount += 1
 97             print "classification error",docList[docIndex]
 98     print the error rate is: ,float(errorCount)/len(testSet)
 99     #return vocabList,fullText
100     
101     
102 def main():
103     spamTest(email/spam/,email/ham/)
104     
105 if __name__ == __main__:
106     main()

 

 

 

 
  • 从个人广告中获取地区倾向

    这个是从某个网站上提取了不同地区板块的信息,分析他们的用词是不是有某些规律

    数据集

    这里的数据使用RSS获取的,用到了python的feedparse包,想了解可以看这里.这里分别获取了某网站两个地区板块中的信息

    测试方法

    交叉验证

    实现细节


    1.这里有两种字符需要特别处理(其实他们有很大重合),一种是频率最高的一些,另一种是所谓的停用词(我的理解其实就是那些使用频率很高但没什么实际意义的),各种语言的停用词可以看这里。 我们需要移除这些词以使得结果更能体现出地区差异。
    2.getTopWords函数实际上就是对这个概率统计了一下特征。对学习贝叶斯来说不是必要代码
    3.除了数据来源不同实现细节和上面的很相似
  1.   1 #coding=utf-8
      2 from numpy import *
      3 
      4 #解析文档的函数
      5 def textParse(bigString):
      6     import re
      7     listOfTokens = re.split(r\W*,bigString)
      8     return [tok.lower() for tok in listOfTokens if len(tok) > 2]
      9     
     10     
     11 #创建一个带有所有单词的列表
     12 def createVocabList(dataSet):
     13     vocabSet = set([])
     14     for document in dataSet:
     15         vocabSet = vocabSet | set(document)
     16     return list(vocabSet)
     17     
     18 def setOfWords2Vec(vocabList, inputSet):
     19     retVocabList = [0] * len(vocabList)
     20     for word in inputSet:
     21         if word in vocabList:
     22             retVocabList[vocabList.index(word)] = 1
     23         else:
     24             print word ,word ,not in dict
     25     return retVocabList
     26 
     27 #另一种模型    
     28 def bagOfWords2VecMN(vocabList, inputSet):
     29     returnVec = [0]*len(vocabList)
     30     for word in inputSet:
     31         if word in vocabList:
     32             returnVec[vocabList.index(word)] += 1
     33     return returnVec
     34 
     35 def trainNB0(trainMatrix,trainCatergory):
     36     numTrainDoc = len(trainMatrix)
     37     numWords = len(trainMatrix[0])
     38     pAbusive = sum(trainCatergory)/float(numTrainDoc)
     39     #防止多个概率的成绩当中的一个为0
     40     p0Num = ones(numWords)
     41     p1Num = ones(numWords)
     42     p0Denom = 2.0
     43     p1Denom = 2.0
     44     for i in range(numTrainDoc):
     45         if trainCatergory[i] == 1:
     46             p1Num +=trainMatrix[i]
     47             p1Denom += sum(trainMatrix[i])
     48         else:
     49             p0Num +=trainMatrix[i]
     50             p0Denom += sum(trainMatrix[i])
     51     p1Vect = log(p1Num/p1Denom)#处于精度的考虑,否则很可能到限归零
     52     p0Vect = log(p0Num/p0Denom)
     53     return p0Vect,p1Vect,pAbusive
     54     
     55 def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
     56     p1 = sum(vec2Classify * p1Vec) + log(pClass1)    #element-wise mult
     57     p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
     58     if p1 > p0:
     59         return 1
     60     else: 
     61         return 0
     62 
     63 def stopWords():
     64     stopW = []
     65     f = open(stopwords.txt).readlines()
     66     for eachLine in f:
     67         stopW.append(eachLine[:-1])
     68     return stopW
     69 
     70 def calcMostFreq(vocabList,fullText):
     71     import operator
     72     freqDict = {}
     73     for token in vocabList:
     74         freqDict[token]=fullText.count(token)
     75     sortedFreq = sorted(freqDict.iteritems(), key=operator.itemgetter(1), reverse=True) 
     76     return sortedFreq[:30]       
     77 
     78 def localWords(rss1,rss0):
     79     import feedparser
     80     feed1 = feedparser.parse(rss1)
     81     feed0 = feedparser.parse(rss0)
     82     docList=[]; classList = []; fullText =[]
     83     minLen = min(len(feed1[entries]),len(feed0[entries]))
     84     for i in range(minLen):
     85         wordList = textParse(feed1[entries][i][summary])
     86         docList.append(wordList)
     87         fullText.extend(wordList)
     88         classList.append(1) #NY is class 1
     89         wordList = textParse(feed0[entries][i][summary])
     90         docList.append(wordList)
     91         fullText.extend(wordList)
     92         classList.append(0)
     93     vocabList = createVocabList(docList)#create vocabulary
     94     top30Words = calcMostFreq(vocabList,fullText)   #remove top 30 words
     95     for pairW in top30Words:
     96         if pairW[0] in vocabList: vocabList.remove(pairW[0])
     97     stopW = stopWords()
     98     for pairW in stopW:
     99         if pairW[0] in vocabList:
    100             vocabList.remove(pairW[0])
    101     trainingSet = range(2*minLen); testSet=[]           #create test set
    102     for i in range(20):
    103         randIndex = int(random.uniform(0,len(trainingSet)))
    104         testSet.append(trainingSet[randIndex])
    105         del(trainingSet[randIndex])  
    106     trainMat=[]; trainClasses = []
    107     for docIndex in trainingSet:#train the classifier (get probs) trainNB0
    108         trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))
    109         trainClasses.append(classList[docIndex])
    110     p0V,p1V,pSpam = trainNB0(array(trainMat),array(trainClasses))
    111     errorCount = 0
    112     for docIndex in testSet:        #classify the remaining items
    113         wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])
    114         if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
    115             errorCount += 1
    116     print the error rate is: ,float(errorCount)/len(testSet)
    117     return vocabList,p0V,p1V
    118 
    119 def getTopWords(ny,sf):
    120     import operator
    121     vocabList,p0V,p1V=localWords(ny,sf)
    122     topNY=[]; topSF=[]
    123     for i in range(len(p0V)):
    124         if p0V[i] > -6.0 : topSF.append((vocabList[i],p0V[i]))
    125         if p1V[i] > -6.0 : topNY.append((vocabList[i],p1V[i]))
    126     sortedSF = sorted(topSF, key=lambda pair: pair[1], reverse=True)
    127     print "SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**"
    128     for item in sortedSF:
    129         print item[0]
    130     sortedNY = sorted(topNY, key=lambda pair: pair[1], reverse=True)
    131     print "NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**"
    132     for item in sortedNY:
    133         print item[0]    
    134     
    135 def main():
    136     #print stopWords()
    137     localWords(http://newyork.craigslist.org/stp/index.rss,http://sfbay.craigslist.org/stp/index.rss)
    138     
    139 if __name__ == __main__:
    140     main()

     



朴素贝叶斯算法的实例

标签:style   blog   http   io   ar   color   os   使用   sp   

原文地址:http://www.cnblogs.com/MrLJC/p/4107117.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!