Python数据分析学习

 1 import pandas as pd
 2 import numpy as np
 3 import seaborn as sns
 4 import matplotlib as mpl
 5 
 6 import matplotlib.pyplot as plt
 7 from IPython.display import display
 8 plt.style.use("fivethirtyeight")
 9 sns.set_style({‘font.sans-serif‘:[‘simhei‘,‘Arial‘]})
10 %matplotlib inline
11 
12 
13 # 检查Python版本
14 from sys import version_info
15 if version_info.major != 3:
16     raise Exception(‘请使用Python 3 来完成此项目‘)

1 # 导入二手房数据
2 lianjia_df = pd.read_csv(‘lianjia.csv‘)
3 display(lianjia_df.head(n=2))

1 # 检查缺失值情况
2 lianjia_df.info()

lianjia_df.describe()

 1 # 添加新特征房屋均价
 2 df = lianjia_df.copy()
 3 df[‘PerPrice‘] = lianjia_df[‘Price‘]/lianjia_df[‘Size‘]
 4 
 5 # 重新摆放列位置
 6 columns = [‘Region‘, ‘District‘, ‘Garden‘, ‘Layout‘, ‘Floor‘, ‘Year‘, ‘Size‘, ‘Elevator‘, ‘Direction‘, ‘Renovation‘, ‘PerPrice‘, ‘Price‘]
 7 df = pd.DataFrame(df, columns = columns)
 8 
 9 # 重新审视数据集
10 display(df.head(n=2))

 1 # 对二手房区域分组对比二手房数量和每平米房价
 2 df_house_count = df.groupby(‘Region‘)[‘Price‘].count().sort_values(ascending=False).to_frame().reset_index()
 3 df_house_mean = df.groupby(‘Region‘)[‘PerPrice‘].mean().sort_values(ascending=False).to_frame().reset_index()
 4 
 5 f, [ax1,ax2,ax3] = plt.subplots(3,1,figsize=(20,15))
 6 sns.barplot(x=‘Region‘, y=‘PerPrice‘, palette="Blues_d", data=df_house_mean, ax=ax1)
 7 ax1.set_title(‘北京各大区二手房每平米单价对比‘,fontsize=15)
 8 ax1.set_xlabel(‘区域‘)
 9 ax1.set_ylabel(‘每平米单价‘)
10 
11 sns.barplot(x=‘Region‘, y=‘Price‘, palette="Greens_d", data=df_house_count, ax=ax2)
12 ax2.set_title(‘北京各大区二手房数量对比‘,fontsize=15)
13 ax2.set_xlabel(‘区域‘)
14 ax2.set_ylabel(‘数量‘)
15 
16 sns.boxplot(x=‘Region‘, y=‘Price‘, data=df, ax=ax3)
17 ax3.set_title(‘北京各大区二手房房屋总价‘,fontsize=15)
18 ax3.set_xlabel(‘区域‘)
19 ax3.set_ylabel(‘房屋总价‘)
20 
21 plt.show()

1 f, [ax1,ax2] = plt.subplots(1, 2, figsize=(15, 5))
2 # 建房面积的分布情况
3 sns.distplot(df[‘Size‘], bins=20, ax=ax1, color=‘r‘)
4 sns.kdeplot(df[‘Size‘], shade=True, ax=ax1)
5 # 建房面积和出售价格的关系
6 sns.regplot(x=‘Size‘, y=‘Price‘, data=df, ax=ax2)
7 plt.show()

1 df.loc[df[‘Size‘]< 10]

df.loc[df[‘Size‘]>1000]

df = df[(df[‘Layout‘]!=‘叠拼别墅‘)&(df[‘Size‘]<1000)]

f, ax1= plt.subplots(figsize=(20,20))
sns.countplot(y=‘Layout‘, data=df, ax=ax1)
ax1.set_title(‘房屋户型‘,fontsize=15)
ax1.set_xlabel(‘数量‘)
ax1.set_ylabel(‘户型‘)
plt.show()

1 df[‘Renovation‘].value_counts()

# 去掉错误数据“南北”，因为爬虫过程中一些信息位置为空，导致“Direction”的特征出现在这里，需要清除或替换
df[‘Renovation‘] = df.loc[(df[‘Renovation‘] != ‘南北‘), ‘Renovation‘]

# 画幅设置
f, [ax1,ax2,ax3] = plt.subplots(1, 3, figsize=(20, 5))
sns.countplot(df[‘Renovation‘], ax=ax1)
sns.barplot(x=‘Renovation‘, y=‘Price‘, data=df, ax=ax2)
sns.boxplot(x=‘Renovation‘, y=‘Price‘, data=df, ax=ax3)
plt.show()

misn = len(df.loc[(df[‘Elevator‘].isnull()), ‘Elevator‘])
print(‘Elevator缺失值数量为：‘+ str(misn))

 1 # 由于存在个别类型错误，如简装和精装，特征值错位，故需要移除
 2 df[‘Elevator‘] = df.loc[(df[‘Elevator‘] == ‘有电梯‘)|(df[‘Elevator‘] == ‘无电梯‘), ‘Elevator‘]
 3 
 4 # 填补Elevator缺失值
 5 df.loc[(df[‘Floor‘]>6)&(df[‘Elevator‘].isnull()), ‘Elevator‘] = ‘有电梯‘
 6 df.loc[(df[‘Floor‘]<=6)&(df[‘Elevator‘].isnull()), ‘Elevator‘] = ‘无电梯‘
 7 
 8 f, [ax1,ax2] = plt.subplots(1, 2, figsize=(20, 10))
 9 sns.countplot(df[‘Elevator‘], ax=ax1)
10 ax1.set_title(‘有无电梯数量对比‘,fontsize=15)
11 ax1.set_xlabel(‘是否有电梯‘)
12 ax1.set_ylabel(‘数量‘)
13 sns.barplot(x=‘Elevator‘, y=‘Price‘, data=df, ax=ax2)
14 ax2.set_title(‘有无电梯房价对比‘,fontsize=15)
15 ax2.set_xlabel(‘是否有电梯‘)
16 ax2.set_ylabel(‘总价‘)
17 plt.show()

1 grid = sns.FacetGrid(df, row=‘Elevator‘, col=‘Renovation‘, palette=‘seismic‘,size=4)
2 grid.map(plt.scatter, ‘Year‘, ‘Price‘)
3 grid.add_legend()

f, ax1= plt.subplots(figsize=(20,5))
sns.countplot(x=‘Floor‘, data=df, ax=ax1)
ax1.set_title(‘房屋户型‘,fontsize=15)
ax1.set_xlabel(‘数量‘)
ax1.set_ylabel(‘户型‘)
plt.show()

 1 """
 2 特征工程
 3 """
 4 # 移除结构类型异常值和房屋大小异常值
 5 df = df[(df[‘Layout‘]!=‘叠拼别墅‘)&(df[‘Size‘]<1000)]
 6 
 7 # 去掉错误数据“南北”，因为爬虫过程中一些信息位置为空，导致“Direction”的特征出现在这里，需要清除或替换
 8 df[‘Renovation‘] = df.loc[(df[‘Renovation‘] != ‘南北‘), ‘Renovation‘]
 9 
10 # 由于存在个别类型错误，如简装和精装，特征值错位，故需要移除
11 df[‘Elevator‘] = df.loc[(df[‘Elevator‘] == ‘有电梯‘)|(df[‘Elevator‘] == ‘无电梯‘), ‘Elevator‘]
12 
13 # 填补Elevator缺失值
14 df.loc[(df[‘Floor‘]>6)&(df[‘Elevator‘].isnull()), ‘Elevator‘] = ‘有电梯‘
15 df.loc[(df[‘Floor‘]<=6)&(df[‘Elevator‘].isnull()), ‘Elevator‘] = ‘无电梯‘
16 
17 # 只考虑“室”和“厅”，将其它少数“房间”和“卫”移除
18 df = df.loc[df[‘Layout‘].str.extract(‘^\d(.*?)\d.*?‘) == ‘室‘]
19 
20 # 提取“室”和“厅”创建新特征
21 df[‘Layout_room_num‘] = df[‘Layout‘].str.extract(‘(^\d).*‘, expand=False).astype(‘int64‘)
22 df[‘Layout_hall_num‘] = df[‘Layout‘].str.extract(‘^\d.*?(\d).*‘, expand=False).astype(‘int64‘)
23 
24 # 按中位数对“Year”特征进行分箱
25 df[‘Year‘] = pd.qcut(df[‘Year‘],8).astype(‘object‘)
26 
27 # 对“Direction”特征
28 d_list_one = [‘东‘,‘西‘,‘南‘,‘北‘]
29 d_list_two = [‘东西‘,‘东南‘,‘东北‘,‘西南‘,‘西北‘,‘南北‘]
30 d_list_three = [‘东西南‘,‘东西北‘,‘东南北‘,‘西南北‘]
31 d_list_four = [‘东西南北‘]    
32 df[‘Direction‘] = df[‘Direction‘].apply(direct_func)
33 df = df.loc[(df[‘Direction‘]!=‘no‘)&(df[‘Direction‘]!=‘nan‘)]
34 
35 # 根据已有特征创建新特征
36 df[‘Layout_total_num‘] = df[‘Layout_room_num‘] + df[‘Layout_hall_num‘]
37 df[‘Size_room_ratio‘] = df[‘Size‘]/df[‘Layout_total_num‘]
38 
39 # 删除无用特征
40 df = df.drop([‘Layout‘,‘PerPrice‘,‘Garden‘],axis=1)
41 
42 # 对于object特征进行onehot编码
43 df,df_cat = one_hot_encoder(df)

1 df[‘Layout‘].value_counts()

1 # 只考虑“室”和“厅”，将其它少数“房间”和“卫”移除
2 df = df.loc[df[‘Layout‘].str.extract(‘^\d(.*?)\d.*?‘) == ‘室‘]
3 
4 # 提取“室”和“厅”创建新特征
5 df[‘Layout_room_num‘] = df[‘Layout‘].str.extract(‘(^\d).*‘, expand=False).astype(‘int64‘)
6 df[‘Layout_hall_num‘] = df[‘Layout‘].str.extract(‘^\d.*?(\d).*‘, expand=False).astype(‘int64‘)

1 # 按中位数对“Year”特征进行分箱
2 df[‘Year‘] = pd.qcut(df[‘Year‘],8).astype(‘object‘)

1 # 对“Direction”特征
2 d_list_one = [‘东‘,‘西‘,‘南‘,‘北‘]
3 d_list_two = [‘东西‘,‘东南‘,‘东北‘,‘西南‘,‘西北‘,‘南北‘]
4 d_list_three = [‘东西南‘,‘东西北‘,‘东南北‘,‘西南北‘]
5 d_list_four = [‘东西南北‘]    
6 df[‘Direction‘] = df[‘Direction‘].apply(direct_func)
7 df = df.loc[(df[‘Direction‘]!=‘no‘)&(df[‘Direction‘]!=‘nan‘)]

1 # 根据已有特征创建新特征
2 df[‘Layout_total_num‘] = df[‘Layout_room_num‘] + df[‘Layout_hall_num‘]
3 df[‘Size_room_ratio‘] = df[‘Size‘]/df[‘Layout_total_num‘]
4 
5 # 删除无用特征
6 df = df.drop([‘Layout‘,‘PerPrice‘,‘Garden‘],axis=1)

1 # 对于object特征进行onehot编码
2 df,df_cat = one_hot_encoder(df)

1 # data_corr 
2 colormap = plt.cm.RdBu
3 plt.figure(figsize=(20,20))
4 # plt.title(‘Pearson Correlation of Features‘, y=1.05, size=15)
5 sns.heatmap(df.corr(),linewidths=0.1,vmax=1.0, square=True, cmap=colormap, linecolor=‘white‘, annot=True)

1 # 转换训练测试集格式为数组
2 features = np.array(features)
3 prices = np.array(prices)
4 
5 # 导入sklearn进行训练测试集划分
6 from sklearn.model_selection import train_test_split
7 features_train, features_test, prices_train, prices_test = train_test_split(features, prices, test_size=0.2, random_state=0)

 1 from sklearn.model_selection import KFold
 2 from sklearn.tree import DecisionTreeRegressor
 3 from sklearn.metrics import make_scorer
 4 from sklearn.model_selection import GridSearchCV
 5 
 6 # 利用GridSearchCV计算最优解
 7 def fit_model(X, y):
 8     """ 基于输入数据 [X,y]，利于网格搜索找到最优的决策树模型"""
 9     
10     cross_validator = KFold(10, shuffle=True)
11     regressor = DecisionTreeRegressor()
12     
13     params = {‘max_depth‘:[1,2,3,4,5,6,7,8,9,10]}
14     scoring_fnc = make_scorer(performance_metric)
15     grid = GridSearchCV(estimator = regressor, param_grid = params, scoring = scoring_fnc, cv = cross_validator)
16 
17     # 基于输入数据 [X,y]，进行网格搜索
18     grid = grid.fit(X, y)
19 #     print pd.DataFrame(grid.cv_results_)
20     return grid.best_estimator_
21 
22 # 计算R2分数
23 def performance_metric(y_true, y_predict):
24     """计算并返回预测值相比于预测值的分数"""
25     from sklearn.metrics import r2_score
26     score = r2_score(y_true, y_predict)
27 
28     return score

 1 import visuals as vs
 2 
 3 # 分析模型
 4 vs.ModelLearning(features_train, prices_train)
 5 vs.ModelComplexity(features_train, prices_train)
 6 
 7 optimal_reg1 = fit_model(features_train, prices_train)
 8 
 9 # 输出最优模型的 ‘max_depth‘ 参数
10 print("最理想模型的参数 ‘max_depth‘ 是 {} 。".format(optimal_reg1.get_params()[‘max_depth‘]))
11 
12 predicted_value = optimal_reg1.predict(features_test)
13 r2 = performance_metric(prices_test, predicted_value)
14 
15 print("最优模型在测试数据上 R^2 分数 {:,.2f}。".format(r2))