[amazonaccess 1]logistic.py 特征提取

learner = ‘log‘
print "Reading dataset..."
train_data = pd.read_csv(‘train.csv‘)
test_data = pd.read_csv(‘test.csv‘)
submit=learner + str(SEED) + ‘.csv‘
#去除ROLE_CODE特征，因为train和test数据需要同时做变换，所以合到一块
all_data = np.vstack((train_data.ix[:,1:-1], test_data.ix[:,1:-1]))
num_train = np.shape(train_data)[0]

# Transform data
print "Transforming data..."
# Relabel the variable values to smallest possible so that I can use bincount
# on them later.
relabler = preprocessing.LabelEncoder()
for col in range(len(all_data[0,:])):
    relabler.fit(all_data[:, col])
    all_data[:, col] = relabler.transform(all_data[:, col])

def group_data(data, degree=3, hash=hash):
    """ 
    numpy.array -> numpy.array
    
    Groups all columns of data into all combinations of triples
    """
    new_data = []
    m,n = data.shape
    for indicies in combinations(range(n), degree):
        #去除ROLE_TITLE和ROLE_FAMILY组合
    if 5 in indicies and 7 in indicies:
        print "feature Xd"
        #去除ROLE_ROLLUP_1和ROLE_ROLLUP_2组合
    elif 2 in indicies and 3 in indicies:
        print "feature Xd"
    else:
            new_data.append([hash(tuple(v)) for v in data[:,indicies]])
    return array(new_data).T

dp = group_data(all_data, degree=2) 
for col in range(len(dp[0,:])):
    relabler.fit(dp[:, col])
    dp[:, col] = relabler.transform(dp[:, col])
    uniques = len(set(dp[:,col]))
    maximum = max(dp[:,col])
    print col
    if maximum < 65534:
        count_map = np.bincount((dp[:, col]).astype(‘uint16‘))
        for n,i in enumerate(dp[:, col]):
            #只有1条数据的标签，合并
            if count_map[i] <= 1:
                dp[n, col] = uniques
            #只有2条数据的标签，合并
            elif count_map[i] == 2:
                dp[n, col] = uniques+1
    else:
        for n,i in enumerate(dp[:, col]):
            if (dp[:, col] == i).sum() <= 1:
                dp[n, col] = uniques
            elif (dp[:, col] == i).sum() == 2:
                dp[n, col] = uniques+1
    print uniques # unique values
    uniques = len(set(dp[:,col]))
    print uniques
    relabler.fit(dp[:, col])
    dp[:, col] = relabler.transform(dp[:, col])

# Collect the training features together
y = array(train_data.ACTION)
X = all_data[:num_train]
X_2 = dp[:num_train]
X_3 = dt[:num_train]

# Collect the testing features together
X_test = all_data[num_train:]
X_test_2 = dp[num_train:]
X_test_3 = dt[num_train:]

X_train_all = np.hstack((X, X_2, X_3))
X_test_all = np.hstack((X_test, X_test_2, X_test_3))

def OneHotEncoder(data, keymap=None):
     """
     OneHotEncoder takes data matrix with categorical columns and
     converts it to a sparse binary matrix.
     
     Returns sparse binary matrix and keymap mapping categories to indicies.
     If a keymap is supplied on input it will be used instead of creating one
     and any categories appearing in the data that are not in the keymap are
     ignored
     """
     if keymap is None:
          keymap = []
          for col in data.T:
               uniques = set(list(col))
               keymap.append(dict((key, i) for i, key in enumerate(uniques)))
     total_pts = data.shape[0]
     outdat = []
     for i, col in enumerate(data.T):
          km = keymap[i]
          num_labels = len(km)
          spmat = sparse.lil_matrix((total_pts, num_labels))
          for j, val in enumerate(col):
               if val in km:
                    spmat[j, km[val]] = 1
          outdat.append(spmat)
     outdat = sparse.hstack(outdat).tocsr()
     return outdat, keymap

# Xts holds one hot encodings for each individual feature in memory
# speeding up feature selection 
Xts = [OneHotEncoder(X_train_all[:,[i]])[0] for i in range(num_features)]

print "Performing greedy feature selection..."
score_hist = []
N = 10
good_features = set([])
# Greedy feature selection loop
while len(score_hist) < 2 or score_hist[-1][0] > score_hist[-2][0]:
    scores = []
    for f in range(len(Xts)):
        if f not in good_features:
            feats = list(good_features) + [f]
            Xt = sparse.hstack([Xts[j] for j in feats]).tocsr()
            score = cv_loop(Xt, y, model, N)
            scores.append((score, f))
            print "Feature: %i Mean AUC: %f" % (f, score)
    good_features.add(sorted(scores)[-1][1])
    score_hist.append(sorted(scores)[-1])
    print "Current features: %s" % sorted(list(good_features))
    
# Remove last added feature from good_features
good_features.remove(score_hist[-1][1])
good_features = sorted(list(good_features))
print "Selected features %s" % good_features
gf = open("feats" + submit, ‘w‘)
print >>gf, good_features
gf.close()
print len(good_features), " features"

print "Performing hyperparameter selection..."
# Hyperparameter selection loop
score_hist = []
Xt = sparse.hstack([Xts[j] for j in good_features]).tocsr()
if learner == ‘NB‘:
    Cvals = [0.001, 0.003, 0.006, 0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.1]
else:
    Cvals = np.logspace(-4, 4, 15, base=2)  # for logistic
for C in Cvals:
    if learner == ‘NB‘:
        model.alpha = C
    else:
        model.C = C
    score = cv_loop(Xt, y, model, N)
    score_hist.append((score,C))
    print "C: %f Mean AUC: %f" %(C, score)
bestC = sorted(score_hist)[-1][1]
print "Best C value: %f" % (bestC)

print "Performing One Hot Encoding on entire dataset..."
Xt = np.vstack((X_train_all[:,good_features], X_test_all[:,good_features]))
Xt, keymap = OneHotEncoder(Xt)
X_train = Xt[:num_train]
X_test = Xt[num_train:]
    
if learner == ‘NB‘:
    model.alpha = bestC
else:
    model.C = bestC

print "Training full model..."
print "Making prediction and saving results..."
model.fit(X_train, y)
preds = model.predict_proba(X_test)[:,1]
create_test_submission(submit, preds)
preds = model.predict_proba(X_train)[:,1]
create_test_submission(‘Train‘+submit, preds)