标签:
from numpy import * import time starttime = time.time() def loadDataSet(): postingList = [[‘my‘, ‘dog‘, ‘has‘, ‘flea‘, ‘problems‘, ‘help‘, ‘please‘], [‘maybe‘, ‘not‘, ‘take‘, ‘him‘, ‘to‘, ‘dog‘, ‘park‘, ‘stupid‘], [‘my‘, ‘dalmation‘, ‘is‘, ‘so‘, ‘cute‘, ‘I‘, ‘love‘, ‘him‘], [‘stop‘, ‘posting‘, ‘stupid‘, ‘worthless‘, ‘garbage‘], [‘mr‘, ‘licks‘, ‘ate‘, ‘my‘, ‘steak‘, ‘how‘, ‘to‘, ‘stop‘, ‘him‘], [‘quit‘, ‘buying‘, ‘worthless‘, ‘dog‘, ‘food‘, ‘stupid‘]] classVec = [0, 1, 0, 1, 0, 1] return postingList, classVec def createVocabList(dataSet): # dataSet = postingList vocabSet = set([]) # vocabSet = set(dataSet) for document in dataSet: vocabSet = vocabSet | set(document) # return list(vocabSet) # createVocabList = list(set(dataSet)) def setOfWords2Vec(vocabList, inputSet): returnVec = [0] * len(vocabList) # [0, 0 , 0 ,0,..] len(vocabList) 0 for word in vocabList: if word in inputSet: returnVec[vocabList.index(word)] = 1 + 1.0 else: returnVec[vocabList.index(word)] = 1.0 print "the word: %s is not in my Vocabulary!" % word return returnVec def txt2trainxy(filename1, filename2): import re reg = re.compile(r‘\W*‘) # # step 1: loading data... print "stet 1: loading data..." from os import listdir ld1 = listdir(‘email/‘ + filename1); ld2 = listdir(‘email/‘ + filename2) filelist = ld1 + ld2 trainy = ((filename1 + ‘\t‘) * len(ld1) + (filename2 + ‘\t‘) * len(ld2)).split() trainx = []; fulltext = []; i = 0 for File in filelist: if i < len(ld1): fr = reg.split(open(‘email/‘ + filename1 + ‘/‘ + File).readlines()[0].lower()) else: fr = reg.split(open(‘email/‘ + filename2 + ‘/‘ + File).readlines()[0].lower()) trainx.append([f for f in fr if len(f) > 2]) # fulltext.extend([f for f in fr if len(f) > 2]) # i += 1 fulltext = list(set(fulltext)) # set of words trainxws = [[list(set(item)).count(strg) + 1.0 for strg in fulltext] for item in trainx] # bag of words trainxwb = [[item.count(strg) + 1.0 for strg in fulltext] for item in trainx] return trainxws, trainxwb, trainy, trainx, fulltext def testx2vec(testx, fulltext): # set of words testxws = [list(set(testx)).count(strg) + 1.0 for strg in fulltext] # # bag of words testxwb = [testx.count(strg) + 1.0 for strg in fulltext] # for word in testx: if word not in fulltext: print "the word: %s is not in my fulltext!" % word return testxws, testxwb def bayes(testx, trainx, trainy, fulltext): print "---Getting Prob..." s = set(trainy); l = len(trainy); r = len(trainx[0]) IDs = [[id for id in range(l) if trainy[id] == item] for item in s] logproby = [log(array(trainy.count(item)) / float(l)) for item in s] numbxv = [sum([trainx[id] for id in ids], 0) for ids in IDs] numbx = [sum([trainx[id] for id in ids]) + 2.0 for ids in IDs] # probx = [numbxv[i] / float(numbx[i]) for i in range(len(s))] logprobx = [[log(p[i]) for i in range(r)] for p in probx] print "---Printing Prob..." #print probx print [fulltext[i] for i in (-array(probx)).argsort()[:,: 5][0]] # argsort() small to big print trainy[IDs[0][0]] print [fulltext[i] for i in (-array(probx)).argsort()[:,: 5][1]] print trainy[IDs[1][0]] """ print IDs print numbxv print logprobx """ # step 4: showing the result... print "---Showing the result..." # set of words sumlogpxws = sum(array(logprobx) * testx, 1) sumlogpxyws = array(sumlogpxws) + array(logproby) #print logprobx print sumlogpxws print sum(array(probx) * testx, 1) bestyws = trainy[IDs[sumlogpxyws.argmax()][0]] print "---From set of words: ", bestyws """ # bag of words sumlogpxwb = sum(array(logprobx) * testxwb, 1) sumlogpxywb = array(sumlogpxwb) + array(logproby) bestywb = trainy[IDs[sumlogpxywb.argmax()][0]] print "---From bag of words: ", bestywb """ return bestyws def main(): # step 1: loading data... trainxws, trainxwb, trainy, trainx, fulltext = txt2trainxy(‘spam‘,‘ham‘) print fulltext # step 2: training... print "step 2: training..." pass # step 3: testing... print "step 3: testing..." print "---Preparing testdata..." import random l = len(trainy) testid = random.sample(range(l), 20) testxxx = [trainxws[i] for i in testid] testyyy = [trainy[i] for i in testid] testtrainxws = [trainxws[i] for i in range(l) if i not in testid] testtrainy = [trainy[i] for i in range(l) if i not in testid] print "---Testing now..." errorcount = 0; p = len(testid) for i in range(p): if bayes(testxxx[i], testtrainxws, testtrainy, fulltext) != testyyy[i]: errorcount += 1 print errorcount print p print "---Errorrate is: ", (errorcount / float(p)) # step 4: showing the result print "step 4: using..." testx = [‘love‘, ‘my‘, ‘dalmation‘] print "the testx is: ", testx print "---Changing testx into vector..." testxws, testxwb = testx2vec(testx, fulltext) #print testxws bayes(testxws, testtrainxws, testtrainy, fulltext) main() """ trainx, trainy = loadDataSet() fulltext = createVocabList(trainx) print fulltext print setOfWords2Vec(fulltext, trainx[0]) trainxws = [] for t in trainx: trainxws.append(setOfWords2Vec(fulltext, t)) testEntry1 = [‘love‘, ‘my‘, ‘dalmation‘] testEntry2 = [‘stupid‘, ‘garbage‘] bayes(testEntry1, trainxws, trainy, fulltext) """
标签:
原文地址:http://www.cnblogs.com/monne/p/4249324.html