集成学习（Ensemble Learning）

from numpy import *

def loadSimpData():
    datMat = matrix([[ 1. ,  2.1],
        [ 2. ,  1.1],
        [ 1.3,  1. ],
        [ 1. ,  1. ],
        [ 2. ,  1. ]])
    classLabels = [1.0, 1.0, -1.0, -1.0, 1.0]
    return datMat,classLabels

def loadDataSet(fileName):      #general function to parse tab -delimited floats
    numFeat = len(open(fileName).readline().split(‘\t‘)) #get number of fields 
    dataMat = []; labelMat = []
    fr = open(fileName)
    for line in fr.readlines():
        lineArr =[]
        curLine = line.strip().split(‘\t‘)
        for i in range(numFeat-1):
            lineArr.append(float(curLine[i]))
        dataMat.append(lineArr)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat

def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):#just classify the data
    retArray = ones((shape(dataMatrix)[0],1))
    if threshIneq == ‘lt‘:
        retArray[dataMatrix[:,dimen] <= threshVal] = -1.0
    else:
        retArray[dataMatrix[:,dimen] > threshVal] = -1.0
    return retArray
    

def buildStump(dataArr,classLabels,D):
    dataMatrix = mat(dataArr); labelMat = mat(classLabels).T
    m,n = shape(dataMatrix)
    numSteps = 10.0; bestStump = {}; bestClasEst = mat(zeros((m,1)))
    minError = inf #init error sum, to +infinity
    for i in range(n):#loop over all dimensions
        rangeMin = dataMatrix[:,i].min(); rangeMax = dataMatrix[:,i].max();
        stepSize = (rangeMax-rangeMin)/numSteps
        for j in range(-1,int(numSteps)+1):#loop over all range in current dimension
            for inequal in [‘lt‘, ‘gt‘]: #go over less than and greater than
                threshVal = (rangeMin + float(j) * stepSize)
                predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)#call stump classify with i, j, lessThan
                errArr = mat(ones((m,1)))
                errArr[predictedVals == labelMat] = 0
                weightedError = D.T*errArr  #calc total error multiplied by D
                #print "split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" % (i, threshVal, inequal, weightedError)
                if weightedError < minError:
                    minError = weightedError
                    bestClasEst = predictedVals.copy()
                    bestStump[‘dim‘] = i
                    bestStump[‘thresh‘] = threshVal
                    bestStump[‘ineq‘] = inequal
    return bestStump,minError,bestClasEst


def adaBoostTrainDS(dataArr,classLabels,numIt=40):
    weakClassArr = []
    m = shape(dataArr)[0]
    D = mat(ones((m,1))/m)   #init D to all equal
    aggClassEst = mat(zeros((m,1)))
    for i in range(numIt):
        bestStump,error,classEst = buildStump(dataArr,classLabels,D)#build Stump
        #print "D:",D.T
        alpha = float(0.5*log((1.0-error)/max(error,1e-16)))#calc alpha, throw in max(error,eps) to account for error=0
        bestStump[‘alpha‘] = alpha  
        weakClassArr.append(bestStump)                  #store Stump Params in Array
        #print "classEst: ",classEst.T
        expon = multiply(-1*alpha*mat(classLabels).T,classEst) #exponent for D calc, getting messy
        D = multiply(D,exp(expon))                              #Calc New D for next iteration
        D = D/D.sum()
        #calc training error of all classifiers, if this is 0 quit for loop early (use break)
        aggClassEst += alpha*classEst
        #print "aggClassEst: ",aggClassEst.T
        aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T,ones((m,1)))
        errorRate = aggErrors.sum()/m
        print "total error: ",errorRate
        if errorRate == 0.0: break
    return weakClassArr,aggClassEst

def adaClassify(datToClass,classifierArr):
    dataMatrix = mat(datToClass)#do stuff similar to last aggClassEst in adaBoostTrainDS
    m = shape(dataMatrix)[0]
    aggClassEst = mat(zeros((m,1)))
    for i in range(len(classifierArr)):
        classEst = stumpClassify(dataMatrix,classifierArr[i][‘dim‘],                                 classifierArr[i][‘thresh‘],                                 classifierArr[i][‘ineq‘])#call stump classify
        aggClassEst += classifierArr[i][‘alpha‘]*classEst
        print aggClassEst
    return sign(aggClassEst)

def plotROC(predStrengths, classLabels):
    import matplotlib.pyplot as plt
    cur = (1.0,1.0) #cursor
    ySum = 0.0 #variable to calculate AUC
    numPosClas = sum(array(classLabels)==1.0)
    yStep = 1/float(numPosClas); xStep = 1/float(len(classLabels)-numPosClas)
    sortedIndicies = predStrengths.argsort()#get sorted index, it‘s reverse
    fig = plt.figure()
    fig.clf()
    ax = plt.subplot(111)
    #loop through all the values, drawing a line segment at each point
    for index in sortedIndicies.tolist()[0]:
        if classLabels[index] == 1.0:
            delX = 0; delY = yStep;
        else:
            delX = xStep; delY = 0;
            ySum += cur[1]
        #draw line from cur to (cur[0]-delX,cur[1]-delY)
        ax.plot([cur[0],cur[0]-delX],[cur[1],cur[1]-delY], c=‘b‘)
        cur = (cur[0]-delX,cur[1]-delY)
    ax.plot([0,1],[0,1],‘b--‘)
    plt.xlabel(‘False positive rate‘); plt.ylabel(‘True positive rate‘)
    plt.title(‘ROC curve for AdaBoost horse colic detection system‘)
    ax.axis([0,1,0,1])
    plt.show()
    print "the Area Under the Curve is: ",ySum*xStep

/**
 * 梯度提升回归树    简单实现
 * @author ysh  1208706282
 *
 */
public class Gbdt {
    static List<Sample> mSamples;
    static List<Double> mTrainTarget;
    static List<Double> mTrainCurTarget;
    static List<Cart> mCarts;
    /**
     * 加载数据   回归树
     * @param path
     * @param regex
     * @throws Exception
     */
    public  void loadData(String path,String regex) throws Exception{
        mSamples = new ArrayList<Sample>();
        mTrainTarget = new ArrayList<Double>();
        mTrainCurTarget = new ArrayList<Double>();
        BufferedReader reader = new BufferedReader(new FileReader(path));
        String line = null;
        String splits[] = null;
        Sample sample = null;
        while(null != (line=reader.readLine())){
            splits = line.split(regex);
            sample = new Sample();
            sample.label = Double.valueOf(splits[0]);
            mTrainTarget.add(sample.label);
            mTrainCurTarget.add(0.0);
            sample.feature = new ArrayList<Double>(splits.length-1);
            for(int i=0;i<splits.length-1;i++){
                sample.feature.add(new Double(splits[i+1]));
            }
            
            mSamples.add(sample);
        }
        reader.close();
    }
    /**
     * 更新训练集目标
     * @author Administrator
     *
     */
    static class Update implements Runnable{
        int from;
        int to;
        Cart cart;
        public Update(int from,int to,Cart cart){
            this.from = from;
            this.to = to;
            this.cart = cart;
        }
        @Override
        public void run() {
            // TODO Auto-generated method stub
            Sample sample = null;
            for(int i=from;i<to;i++){
                sample = mSamples.get(i);
                mTrainCurTarget.set(i, mTrainCurTarget.get(i)+cart.classify(sample));
                sample.label = mTrainTarget.get(i)-mTrainCurTarget.get(i);
            }
        }
        
    }
    public void updateTarget(Cart cart) throws InterruptedException{
        /*Sample sample = null;
        for(int i=0;i<mSamples.size();i++){
            sample = mSamples.get(i);
            mTrainCurTarget.set(i, mTrainCurTarget.get(i)+cart.classify(sample));
            sample.label = mTrainTarget.get(i)-mTrainCurTarget.get(i);
        }*/
        Update update = null;
        int num = 10;
        Thread ths[] = new Thread[num];
        int size = mSamples.size();
        
        for(int i=0;i<num;i++){
            update = new Update(i*size/num,(i+1)*size/num,cart);
            ths[i] = new Thread(update);
            ths[i].start();
        }
        for(int i=0;i<num;i++){
            ths[i].join();
        }
    }
    public void train(int iters) throws InterruptedException{
        mCarts = new ArrayList<Cart>(iters);
        Random ran = new Random();
        ran.setSeed(100);
        for(int iter=0;iter<iters;iter++){
            System.out.println("start iter "+iter+"  time:"+System.currentTimeMillis()/1000);
            Cart cart = new Cart();
            cart.mFeatureRate = 0.8;
            cart.mMaxDepth = 5;
            cart.mMinLeaf = 1;
            cart.mRandom = ran;
            cart.setData(mSamples);
            cart.train();
            mCarts.add(cart);
            updateTarget(cart);
            System.out.println("end iter "+iter+"  time:"+System.currentTimeMillis()/1000);
        }
    }
    public double classify(Sample sample){
        double ret = 0;
        for(Cart cart:mCarts){
            ret += cart.classify(sample);
        }
        return ret;
    }
    /**
     * @param args
     * @throws Exception 
     */
    public static void main(String[] args) throws Exception {
        // TODO Auto-generated method stub
        System.out.println(System.currentTimeMillis());
        Gbdt gbdt = new Gbdt();
        gbdt.loadData("F:/2016-contest/20161001/train_data_1.csv", ",");
        gbdt.train(100);
        List<Sample> samples = Cart.loadTestData("F:/2016-contest/20161001/valid_data_1.csv", true, ",");
        double sum = 0;
        for(Sample s:samples){
            double val = gbdt.classify(s);
            sum += (val-s.label)*(val-s.label);
            System.out.println(val+"  "+s.label);
        }
        System.out.println(sum/samples.size()+"  "+sum);
        System.out.println(System.currentTimeMillis());
    }

}

#ifndef _DECISION_TREE_H_
#define _DECISION_TREE_H_
#include <string>
#include <vector>
#include <set>
#include <ctime> 
#include <algorithm>
#include <cmath>

using namespace std;

//the data structure for a tuple
struct TupleData
{
vector<int> A;
char label;
};

struct TNode
{
int attrNum;    
int attr;    
char label;
};

struct decision_tree
{
TNode node;
vector<decision_tree*> childs;
};

void init(char * trainname, char * testname);
int readData(vector<TupleData> &data, const char* fileName);
int stringtoint(string s);
void sub_init();
void calculate_ArrtNum();
void calculate_attributes();
void RandomSelectData(vector<TupleData> &data, vector<TupleData> &subdata);
double Entropy(double p, double s);
int creat_classifier(decision_tree *&p, const vector<TupleData> &samples, vector<int> &attributes);
int BestGainArrt(const vector<TupleData> &samples, vector<int> &attributes);
bool Allthesame(const vector<TupleData> &samples, char ch);
char Majorityclass(const vector<TupleData> &samples);
void RandomSelectAttr(vector<int> &data, vector<int> &subdata);
char testClassifier(decision_tree *p, TupleData d);
void testData();
void freeClassifier(decision_tree *p);
void freeArrtNum();
void showResult();
#endif //_DECISION_TREE_H_


#include <iostream>
#include <fstream>
#include <sstream>
#include "random_forest.h"

using namespace std;

vector<decision_tree*> alltrees;

vector<TupleData>    trainAll,
                    train,    
test;    

vector<int>     attributes;    

int trainAllNum=0;    
int testAllNum=0;    
int MaxAttr;    
int *ArrtNum;
unsigned int F;
int tree_num=100;
const int leafattrnum=-1;
int TP=0,
FN=0,
FP=0,
TN=0,
TestP=0,
TestN=0;

void init(char * trainname, char * testname)
{
trainAllNum=readData(trainAll, trainname);
testAllNum=readData(test, testname);
calculate_attributes();
double temp=(double)trainAllNum;
temp=log(temp)/log(2.0);
//     F=round(temp)+1;
F = (unsigned int)floor(temp+0.5)+1;
if(F>MaxAttr) F=MaxAttr;
//cout<<"f="<<F<<endl;
}

void sub_init()
{
RandomSelectData(trainAll, train);
calculate_ArrtNum();
}


int readData(vector<TupleData> &data, const char* fileName)
{
ifstream fin;
fin.open(fileName);
string line;

int datanum=0;

while(getline(fin,line))
{
TupleData d;
        istringstream stream(line);
        string str;
while(stream>>str)
{
if(str.find(‘+‘)==0)
{
d.label=‘+‘;
}
else if(str.find(‘-‘)==0)
{
 d.label=‘-‘;
}
else
{
int j=stringtoint(str);
d.A.push_back(j);
}
}

data.push_back(d);    
datanum++;
}

fin.close();
return datanum;
}

void RandomSelectData(vector<TupleData> &data, vector<TupleData> &subdata)
{
int index;
subdata.clear();
int d=0;
while (d < trainAllNum)
{
index = rand() % trainAllNum;
subdata.push_back(data.at(index));
d++;
}
}

void calculate_attributes()
{
TupleData d=trainAll.at(0);
MaxAttr=d.A.size();
attributes.clear();

for (int i = 0; i < MaxAttr; i++)
{
attributes.push_back(i);
}

ArrtNum=new int[MaxAttr];
}


int stringtoint(string s)
{
int sum=0;
for(int i=0; s[i]!=‘\0‘;i++)
{
int j=int(s[i])-48;
sum=sum*10+j;
}
return sum;
}

void calculate_ArrtNum()
{
for(int i=0; i<MaxAttr;i++) ArrtNum[i]=0;
for (vector<TupleData>::const_iterator it = train.begin(); it != train.end(); it++)    
{
int i=0;
for (vector<int>::const_iterator intt=(*it).A.begin(); intt!=(*it).A.end();intt++)
{
int valuemax=(*intt)+1;   //(*it).A.at(i)???
if(valuemax>ArrtNum[i]) ArrtNum[i]=valuemax;
i++;
}
}
}


double Entropy(double p, double s)
{
double n = s - p;
double result = 0;
if (n != 0)
result += - double(n) / s * log(double(n) / s) / log(2.0);
if (p != 0)
result += double(-p) / s * log(double(p) / s) / log(2.0);
return result;
}

int creat_classifier(decision_tree *&p, const vector<TupleData> &samples, vector<int> &attributes)
{
if (p == NULL)
p = new decision_tree();
if (Allthesame(samples, ‘+‘))
{
p->node.label = ‘+‘;
p->node.attrNum = leafattrnum;
p->childs.clear();
return 1;
}
if (Allthesame(samples, ‘-‘))
{
p->node.label = ‘-‘;
p->node.attrNum = leafattrnum;
p->childs.clear();
return 1;
}
if (attributes.size() == 0)
{
p->node.label = Majorityclass(samples);
p->node.attrNum = leafattrnum;
p->childs.clear();
return 1;
}
p->node.attrNum = BestGainArrt(samples, attributes);

p->node.label = ‘ ‘;

vector<int> newAttributes;
for (vector<int>::iterator it = attributes.begin(); it != attributes.end(); it++)
if ((*it) != p->node.attrNum)
newAttributes.push_back((*it));

int maxvalue=ArrtNum[p->node.attrNum];
vector<TupleData>* subSamples = new vector<TupleData>[maxvalue];
for (int i = 0; i < maxvalue; i++)
subSamples[i].clear();

for (vector<TupleData>::const_iterator it = samples.begin(); it != samples.end(); it++)
{
subSamples[(*it).A.at(p->node.attrNum)].push_back((*it));
}

decision_tree *child;
for (int i = 0; i < maxvalue; i++)
{
child = new decision_tree;
child->node.attr = i;
if (subSamples[i].size() == 0)
child->node.label = Majorityclass(samples);
else
creat_classifier(child, subSamples[i], newAttributes);
p->childs.push_back(child);
}
delete[] subSamples;
return 0;
}

int BestGainArrt(const vector<TupleData> &samples, vector<int> &attributes)
{
int attr, 
bestAttr = 0,
p = 0,
s = (int)samples.size();

for (vector<TupleData>::const_iterator it = samples.begin(); it != samples.end(); it++)
{
if ((*it).label == ‘+‘)
p++;
}

double infoD;
double bestResult = 0;
infoD=Entropy(p, s);

vector<int> m_attributes;
RandomSelectAttr(attributes, m_attributes);

for (vector<int>::iterator it = m_attributes.begin(); it != m_attributes.end(); it++)
{
attr = (*it);
double result = infoD;

int maxvalue=ArrtNum[attr];
int* subN = new int[maxvalue];
int* subP = new int[maxvalue];
int* sub = new int[maxvalue];
for (int i = 0; i < maxvalue; i++)
{
subN[i] = 0;
subP[i] = 0;
sub[i]=0;
}
for (vector<TupleData>::const_iterator jt = samples.begin(); jt != samples.end(); jt++)
{
if ((*jt).label == ‘+‘)
subP[(*jt).A.at(attr)] ++;
else
subN[(*jt).A.at(attr)] ++;
sub[(*jt).A.at(attr)]++;
}

double SplitInfo=0;
for(int i=0; i<maxvalue; i++)
{
double partsplitinfo;
partsplitinfo=-double(sub[i])/s*log(double(sub[i])/s)/log(2.0);
SplitInfo=SplitInfo+partsplitinfo;
}

double infoattr=0;
for (int i = 0; i < maxvalue; i++)
{
double partentropy;
partentropy=Entropy(subP[i], subP[i] + subN[i]);
infoattr=infoattr+((double)(subP[i] + subN[i])/(double)(s))*partentropy;
}
result=result-infoattr;
result=result/SplitInfo;

if (result > bestResult)
{
bestResult = result;
bestAttr = attr;
}
delete[] subN;
delete[] subP;
delete[] sub;
}

if (bestResult == 0)
{
bestAttr=attributes.at(0);
}
return bestAttr;
}

void RandomSelectAttr(vector<int> &data, vector<int> &subdata)
{
int index;
unsigned int dataNum=data.size();
subdata.clear();
if(dataNum<=F)
{
for (vector<int>::iterator it = data.begin(); it != data.end(); it++)
{
int attr = (*it);
subdata.push_back(attr);
}
}
else
{
set<int> AttrSet;
AttrSet.clear();
while (AttrSet.size() < F)
{
index = rand() % dataNum;
if (AttrSet.count(index) == 0)
{
AttrSet.insert(index);
subdata.push_back(data.at(index));
}
}
}
}

bool Allthesame(const vector<TupleData> &samples, char ch)
{
for (vector<TupleData>::const_iterator it = samples.begin(); it != samples.end(); it++)
if ((*it).label != ch)
return false;
return true;
}

char Majorityclass(const vector<TupleData> &samples)
{
int p = 0, n = 0;
for (vector<TupleData>::const_iterator it = samples.begin(); it != samples.end(); it++)
if ((*it).label == ‘+‘)
p++;
else
n++;
if (p >= n)
return ‘+‘;
else
return ‘-‘;
}

char testClassifier(decision_tree *p, TupleData d)
{
if (p->node.label != ‘ ‘)
return p->node.label;
int attrNum = p->node.attrNum;
if (d.A.at(attrNum) < 0)
return ‘ ‘;
return testClassifier(p->childs.at(d.A.at(attrNum)), d);
}

void testData()
{
for (vector<TupleData>::iterator it = test.begin(); it != test.end(); it++)
{
if((*it).label==‘+‘) TestP++;
else TestN++;

int p=0, n=0;
for(int i=0;i<tree_num;i++)
{
if(testClassifier(alltrees.at(i), (*it))==‘+‘)  p++;
else n++;
}

if(p>n)
{
if((*it).label==‘+‘) TP++;
else FP++;
}
else
{
if((*it).label==‘+‘) FN++;
else TN++;
}
}
}


void freeClassifier(decision_tree *p)
{
if (p == NULL)
return;
for (vector<decision_tree*>::iterator it = p->childs.begin(); it != p->childs.end(); it++)
{
freeClassifier(*it);
}
delete p;
}

void freeArrtNum()
{
delete[] ArrtNum;
}

void showResult()
{
cout<<"Train size:    "<< trainAllNum<<endl;
cout<<"Test size:    "<<testAllNum<<endl;    
cout << "True positive:    " << TP << endl;
cout << "False negative:    "<< FN<<endl;
cout << "False positive:    "<<FP<<endl;
cout << "True negative:    "<<TN<<endl;

//     cout << TP << endl;
//     cout << FN<<endl;
//     cout <<FP<<endl;
//     cout <<TN<<endl;
}

int main(int argc, char **argv)
{
char * trainfile=argv[1];
char * testfile=argv[2];

//cout<<"input the F and tree_num"<<endl;
//cin>>F>>tree_num;

srand((unsigned)time(NULL)); 

init(trainfile, testfile);

for(int i=0; i<tree_num; i++)
{
sub_init();
decision_tree * root=NULL;
creat_classifier(root, train, attributes);
alltrees.push_back(root);
}

testData();

for (vector<decision_tree *>::const_iterator it = alltrees.begin(); it != alltrees.end(); it++)
{
freeClassifier((*it));
}

freeArrtNum();

showResult();
return 0;
}