标签：The 文本处理 split rds 训练 lis bag cab mtr
--import numpy as np
import matplotlib.pyplot as plt
from numpy import *
"""
function:
创建数据集
parameters:
无
returns:
postingList - 数据集
classVec - 标签集
"""
def loadDataSet():
postingList = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],
['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'him'],
['stop', 'posting', 'stupid', 'worthless', 'garbage'],
['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
classVec = [0,1,0,1,0,1]
return postingList, classVec
"""
function:
从数据集中提取词汇表(不重复)
parameters:
dataSet - 数据集
retunrns:
vocalSet - 不重复的词汇表
"""
def createVocabList(dataSet):
vocabSet = set([])
for document in dataSet:
vocabSet = vocabSet | set(document)
return list(vocabSet)
"""
function:
根据之前创建的词汇表来对输入数据进行向量化
parameters:
vocabList - 词汇表
inputSet - 输入的一个文档
returns:
returnVec - 该文档向量
"""
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0]*len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:#前面创建词汇表有问题
print("the word: %s is not in my Vocalbulary" % word)
return returnVec
"""
function:
朴素贝叶斯分类器训练函数
parameters:
trainMatrix - 训练文档矩阵,每篇文档调用set0OfWord2Vec生成的returnVec组成的矩阵
trainCategory - 标签向量
returns:
p0Vec - 侮辱类的条件概率数组，即每个词汇属于侮辱类的概率
p1Vect - 非侮辱类的条件概率数组，即每个词汇属于非侮辱类的概率
pAbusive - 文档属于侮辱类的概率
"""
def trainNB0(trainMatrix, trainCategory):
numTrainDocs = len(trainMatrix)#文档数目
numWords = len(trainMatrix[0])#每篇文档的词条数
pAbusive = sum(trainCategory)/float(numTrainDocs)#文档属于侮辱类的概率
#词条出现数初始化
p0Num = ones(numWords)
p1Num = ones(numWords)
#分母初始化
p0Denom = 2.0
p1Denom = 2.0
for i in range(numTrainDocs):#对每篇文档
if trainCategory[i] == 1:#统计侮辱类的条件概率
p1Num += trainMatrix[i]
p1Denom += sum(trainMatrix[i])#这里我是理解成全概率公式
else:
p0Num += trainMatrix[i]
p0Denom += sum(trainMatrix[i])
p0Vect = log(p0Num/p0Denom)
p1Vect = log(p1Num/p1Denom)
return p0Vect, p1Vect, pAbusive
"""
function:
朴素贝叶斯分类函数
parameters:
vec2Classify - 待分类向量
p0Vec - 侮辱类的条件概率数组
p1Vec - 非侮辱类的条件概率数组
pClass1 - 文档属于侮辱类的概率
returns:
"""
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
#对数相加，相当与计算两个向量相乘的结果
#这里不太懂，感觉和公式对不上
p1 = sum(vec2Classify * p1Vec) + log(pClass1)
p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
if __name__ == "__main__":
listOPosts, listClasses = loadDataSet()
myVocabList = createVocabList(listOPosts)
trainMat = []
for postinDoc in listOPosts:
trainMat.append(setOfWords2Vec(myVocabList, postinDoc))
p0V, p1V, pAb = trainNB0(array(trainMat), array(listClasses))
testEntry = ['love', 'my', 'dalmation']
thisDoc = array(setOfWords2Vec(myVocabList, testEntry))
print(testEntry," classified as: ",classifyNB(thisDoc,p0V,p1V,pAb))
testEntry = ['stupid', 'garbage']
thisDoc = array(setOfWords2Vec(myVocabList,testEntry))
print(testEntry," classified as: ",classifyNB(thisDoc,p0V,p1V,pAb))
一个小优化，相比与之前只统计词出现与否的词条模型，词袋模型统计词出现的次数
-不清楚为什么我做出来的错误率这么高，算了，先放着吧
import re
import random
from array import *
import numpy as np
from numpy import *
"""
function:
从数据集中提取词汇表(不重复)
parameters:
dataSet - 数据集
retunrns:
vocalSet - 不重复的词汇表
"""
def createVocabList(dataSet):
vocabSet = set([])
for document in dataSet:
vocabSet = vocabSet | set(document)
return list(vocabSet)
"""
function:
根据之前创建的词汇表来对输入数据进行向量化
parameters:
vocabList - 词汇表
inputSet - 输入的一个文档
returns:
returnVec - 该文档向量
"""
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0]*len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:#前面创建词汇表有问题
print("the word: %s is not in my Vocalbulary" % word)
return returnVec
"""
function:
朴素贝叶斯分类器训练函数
parameters:
trainMatrix - 训练文档矩阵,每篇文档调用set0OfWord2Vec生成的returnVec组成的矩阵
trainCategory - 标签向量
returns:
p0Vec - 侮辱类的条件概率数组，即每个词汇属于侮辱类的概率
p1Vect - 非侮辱类的条件概率数组，即每个词汇属于非侮辱类的概率
pAbusive - 文档属于侮辱类的概率
"""
def trainNB0(trainMatrix, trainCategory):
numTrainDocs = len(trainMatrix)#文档数目
numWords = len(trainMatrix[0])#每篇文档的词条数
pAbusive = sum(trainCategory)/float(numTrainDocs)#文档属于侮辱类的概率
#词条出现数初始化
p0Num = ones(numWords)
p1Num = ones(numWords)
#分母初始化
p0Denom = 2.0
p1Denom = 2.0
for i in range(numTrainDocs):#对每篇文档
if trainCategory[i] == 1:#统计侮辱类的条件概率
p1Num += trainMatrix[i]
p1Denom += sum(trainMatrix[i])#这里我是理解成全概率公式
else:
p0Num += trainMatrix[i]
p0Denom += sum(trainMatrix[i])
p0Vect = log(p0Num/p0Denom)
p1Vect = log(p1Num/p1Denom)
return p0Vect, p1Vect, pAbusive
"""
function:
朴素贝叶斯分类函数
parameters:
vec2Classify - 待分类向量
p0Vec - 侮辱类的条件概率数组
p1Vec - 非侮辱类的条件概率数组
pClass1 - 文档属于侮辱类的概率
returns:
"""
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
#对数相加，相当与计算两个向量相乘的结果
#这里不太懂，感觉和公式对不上
p1 = sum(vec2Classify * p1Vec) + log(pClass1)
p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
"""
function:
处理文本
parameter:
bigString - 文本
return:
tok - 文本处理后所得词汇向量
"""
def textParse(bigString):
listOfTokens = re.split(rb'\w*', bigString)
return [tok.lower() for tok in listOfTokens if len(tok) > 2]
"""
function:
垃圾邮件过滤
parameter:
无
returns:
无
"""
def spamTest():
docList = [] #数据集
classList = [] #标签集
fullText = [] #???
for i in range(1,26):
wordList = textParse(open('spam/%d.txt' % i, 'rb').read())
docList.append(wordList)
fullText.extend(wordList)
classList.append(1)
wordList = textParse(open('ham/%d.txt' % i, 'rb').read())
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
vocabList = createVocabList(docList)#创建词汇表
trainingSet = list(range(50))#训练集
testSet = []#测试集
for i in range(10):#选取10个测试集，并从训练集中删除
randIndex = int(random.uniform(0,len(trainingSet)))
testSet.append(trainingSet[randIndex])
del(trainingSet[randIndex])#???
trainMat = []#数字化的训练集
trainClasses = []#标签集
for docIndex in trainingSet:#遍历训练集,计算数据
trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))
trainClasses.append(classList[docIndex])
p0V, p1V, pSpam = trainNB0(array(trainMat), array(trainClasses))
errorCount = 0
for docIndex in testSet:#测试集计算错误率
wordVector= setOfWords2Vec(vocabList, docList[docIndex])
if classifyNB(array(wordVector), p0V, p1V, pSpam) != classList[docIndex]:
errorCount += 1
print("the error rate is: ", float(errorCount)/len(testSet))
if __name__ == "__main__":
spamTest()最后一个不写了
标签：The 文本处理 split rds 训练 lis bag cab mtr
原文地址：https://www.cnblogs.com/w-j-c/p/10436201.html
