##########################
# KNN: k Nearest Neighbors
#输入：newInput: (1xN)的待分类向量
#         dataSet:   (NxM)的训练数据集
#         labels:      训练数据集的类别标签向量
#         k:             近邻数
# 输出：可能性最大的分类标签
##########################

from numpy import
import operator

#创建一个数据集，包含2个类别共4个样本
def createDataSet():
        # 生成一个矩阵，每行表示一个样本
        group = array([[1.0,0.9],[1.0,1.0],[0.1,0.2],[0.0,0.1]])
        # 4个样本分别所属的类别
        labels = [‘A‘, ‘A‘, ‘B‘, ‘B‘]
        return group, labels

# KNN分类算法函数定义
def KNNClassify(newInput, dataSet, labels, k)：
        numSamples = dataSet.shape[0]  #shape[0]表示行数
      
        ## step1：计算距离
        # tile(A, reps)：构造一个矩阵，通过A重复reps次得到
        # the following copy numSamples rows for dataSet
        diff = tile(newInput, (numSamples, 1)) -dataSet #按元素求差值
        squareDiff = diff ** 2 #将差值平方
        squareDist = sum(squaredDiff, axis = 1) # 按行累加

        ##step2：对距离排序
        # argsort() 返回排序后的索引值
        sortedDistIndices = argsort(distance)
        classCount = {} # define a dictionary (can be append element)
        for i in xrange(k):
             ##step 3: 选择k个最近邻
             voteLabel = labels[sortedDistIndices[i]]

             ## step 4:计算k个最近邻中各类别出现的次数
             # when the key voteLabel is not in dictionary classCount，get()
             # will return 0
             classCount[voteLabel] = classCount.get(voteLabel, 0) + 1

##step 5：返回出现次数最多的类别标签
maxCount = 0
for key, value in classCount.items():
         if value > maxCount:
             maxCount = value
             maxIndex = key

         return maxIndex

import KNN
from numpy import * 
#生成数据集和类别标签
dataSet,labels = KNN.createDataSet()
#定义一个未知类别的数据
testX = array([1.2, 1.0])
k=3
#调用分类函数对未知数据分类
outputLabel = KNN.KNNClassify(testX, dataSet, labels, 3)
print "Your input is:", testX, " and classified to class:", outputLabel

testX = array([0.1, 0.3])
outputLabel = KNN.KNNClassify(testX,dataSet, labels, 3)
print "Your input is:", testX, "and classified to class:", outputLabel

Your input is: [1.2 1.0] and classified to class: A
Your input is: [0.1 0.3] and classified to class: B

#过滤垃圾邮件
def textParse(bigString):      #正则表达式进行文本解析
      import re
      listOfTokens = re.split(r‘\W*‘, bigString)
      return [tok.lower() for tok in listOfTokens if len(tok) > 2]

def spamTest()
      docList = []; classList = []; fullText = []
      for i in range(1,26):     #导入并解析文本文件
             wordList = textParse(open(‘email/spam/%d.txt‘%i).read())
             docList.append(wordList)
             fullText.extend(wordList)
             classList.append(1)
             wordList = textParse(open(‘email/ham/%d.txt‘%i).read())
             docList.append(wordList)
             fullText.extend(wordList)
             classList.append(0)
      vocabList = createVocabList(docList)
      trainingSet = range(50);testSet = []
      for i in range(10):                  #随机构建训练集
             randIndex = int(random.uniform(0, len(trainingSet)))
             testSet.append(trainingSet[randIndex]) #随机挑选一个文档索引号放入测试集
              del(trainingSet[randIndex])  #将该文档索引号从训练集中剔除
       trainMat = []; trainClasses = []
       for docIndex in trainingSet:
              trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))
              trainClasses.append(classList[docIndex])
       p0V,p1V,pSpam = trainNBO(array(trainMat), array(trainClasses))
       errorCount = 0
       for docIndex in testSet:        #对测试集进行分类
             wordVector = setOfWords2Vec(vocabList, docList[docIndex])
             if classifyNB(array(wordVector), p0V,p1V != classList[docIndex]:
                 errorCount +=1
        print ‘the error rate is:‘, float(errorCount)/len(testSet)

def loadDataSet():
       dataMat = []; labelMat = []
        fr = open(‘d:/testSet.txt‘)
        for line in fr.readlines():
               lineArr = line.strip().split()
               dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])])
               labelMat.append(int(lineArr[2]))
         return dataMat, labelMat

def sigmoid(inX):
      return 1.0/(1+exp(-inX))

def gradAscent(dataMatln, classLabels):
      dataMatrix = mat(dataMatln)  # 将先验数据集转换为NumPy矩阵
      labelMat = mat(classLabels).transpose() #将先验数据的类标签转换为NumPy矩阵
   
      m,n = shape(dataMatrix)
      alpha = 0.001     #设置逼近步长调整系数
      maxCycles = 500  #设置最大迭代次数为500
      weights = ones((n,1)) #weights即为需要迭代求解的参数向量

      for k in range(maxCycles): #heavy on matrix operations
            h = sigmoid(dataMatrix * weights) #代入样本向量求得“样本y” sigmoid转换值
            error = (labelMat - h)    #求差
            weights = weights + alpha * dataMatrix.transpose() * error #根据差值调整参数向量
      return weights

1.5 50 thin
1.5 60 fat
1.6 40 thin
1.6 60 fat
1.7 60 thin
1.7 80 fat
1.8 60 thin
1.8 90 fat
1.9 70 thin
1.9 80 fat

# -*- coding: utf-8 -*-
import numpy as np
import scipy as sp
from sklearn import tree
from sklearn.metrics import precision recall curve
from sklearn.metrics import classification report
from sklearn.cross_validation import train_test_split

"数据读入"
data   = []
labels = []
with open("d:\\python\\ml\\data\\1.txt") as ifile:
       for line in ifile:
              tokens = line.strip().split(‘ ‘)
              data.append([float(tk) for tk in tokens[:-1]])
              labels.append(tokens[-1])
x = np.array(data)
labels = np.array(labels)
y = np.zeros(labels.shape)

"‘ 标签转化为0/1‘"
y[labels == ‘fat‘] =1

"‘ 拆分训练数据与测试数据 ‘"
x_train, x_test, y_train, y_test = train_test_split(x,y,test_size = 0.2)

"‘ 使用信息熵作为划分标准，对决策树进行训练 ‘"
clf = tree.DecisionTreeClassifier(criterion=‘entropy‘)
print(clf)
clf.fit(x_train, y_train)

"‘ 把决策树结构写入文件 ‘"
with open("tree.dot", ‘w‘) as f:
       f=tree.export_graphviz(clf, out_file=f)

"‘ 系数反映每个特征的影响力。越大表示该特征在分类中起到的作用越大 ‘"
print(clf.feature_importances_)

"‘ 测试结果的打印 ‘"
answer = clf.predict(x_train)
print(x_train)
print(answer)
print(y_train)
print(np.mean(answer == y_train))

"‘ 准备率与召回率 ‘"
precision,recall,thresholds = precision_recall_curve(y_train)
answer = clf.predict_proba(x)[:,1]
print(classification_report(y,answer,target_names = [‘thin‘, ‘fat‘]))

[ 0.2488562  0.7511438]
array([[  1.6,  60. ],
       [  1.7,  60. ],
       [  1.9,  80. ],
       [  1.5,  50. ],
       [  1.6,  40. ],
       [  1.7,  80. ],
       [  1.8,  90. ],
       [  1.5,  60. ]])
array([ 1.,  0.,  1.,  0.,  0.,  1.,  1.,  1.])
array([ 1.,  0.,  1.,  0.,  0.,  1.,  1.,  1.])
1.0

             precision    recall  f1-score   support
       thin       0.83      1.00      0.91         5
        fat        1.00      0.80      0.89         5
avg / total       1.00      1.00      1.00         8

array([ 0.,  1.,  0.,  1.,  0.,  1.,  0.,  1.,  0.,  0.])
array([ 0.,  1.,  0.,  1.,  0.,  1.,  0.,  1.,  0.,  1.])

可以看到，对训练过的数据做测试，准确率是100%。但是最后将所有数据进行测试，会出现1个测试样本分类错误。

说明本例的决策树对训练集的规则吸收的很好，但是预测性稍微差点。

digraph Tree {
0 [label="X[1] <= 55.0000\nentropy = 0.954434002925\nsamples = 8", shape="box"] ;
1 [label="entropy = 0.0000\nsamples = 2\nvalue = [ 2.  0.]", shape="box"] ;
0 -> 1 ;
2 [label="X[1] <= 70.0000\nentropy = 0.650022421648\nsamples = 6", shape="box"] ;
0 -> 2 ;
3 [label="X[0] <= 1.6500\nentropy = 0.918295834054\nsamples = 3", shape="box"] ;
2 -> 3 ;
4 [label="entropy = 0.0000\nsamples = 2\nvalue = [ 0.  2.]", shape="box"] ;
3 -> 4 ;
5 [label="entropy = 0.0000\nsamples = 1\nvalue = [ 1.  0.]", shape="box"] ;
3 -> 5 ;
6 [label="entropy = 0.0000\nsamples = 3\nvalue = [ 0.  3.]", shape="box"] ;
2 -> 6 ;
}