机器学习期末项目

点击查看代码。

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import graphviz
from time import time
import datetime
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import KBinsDiscretizer
from sklearn.preprocessing import StandardScaler
from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import precision_score, recall_score
from sklearn.linear_model import LogisticRegression as LR
from sklearn.metrics import mean_squared_error
from sklearn.svm import SVC

# 独热编码表头
COLUMNS1 = ['Ville_id', 'Age', 'Married', 'Number_children', 'education_level', 'total_members', 'gained_asset',
            'durable_asset', 'save_asset', 'living_expenses', 'other_expenses', 'incoming_salary', 'incoming_own_farm',
            'incoming_business', 'incoming_no_business', 'incoming_agricultural', 'farm_expenses', 'labor_primary',
            'lasting_investment',
            'no_lasting_investmen', 'depressed', 'sex_0', 'sex_1']
COLUMNS1_1 = ["sex"]

# 分箱表头
COLUMNS2 = ['Ville_id', 'Married', 'Number_children', 'education_level', 'total_members', 'incoming_salary',
            'incoming_own_farm', 'incoming_business', 'incoming_no_business', 'labor_primary', 'sex_0', 'sex_1',
            'depressed', 'Age', 'gained_asset', 'durable_asset', 'save_asset', 'living_expenses', 'other_expenses',
            'incoming_agricultural', 'farm_expenses', 'lasting_investment', 'no_lasting_investmen']
COLUMNS2_2 = ['Age', 'gained_asset', 'durable_asset', 'save_asset', 'living_expenses', 'other_expenses',
              'incoming_agricultural', 'farm_expenses', 'labor_primary', 'lasting_investment', 'no_lasting_investmen']

# 随机森林及决策树参数范围
parameter_grid = {'max_depth': np.arange(5, 8, 1), 'max_features': np.arange(7, 10, 1)}
max_depthl = []
max_featuresl = []
# 决策树可视化特征名称
feature_name = ['Ville_id', 'Married', 'Number_children', 'education_level', 'total_members', 'incoming_salary',
                'incoming_own_farm', 'incoming_business', 'incoming_no_business', 'labor_primary', 'sex_0', 'sex_1',
                'Age', 'gained_asset', 'durable_asset', 'save_asset', 'living_expenses', 'other_expenses',
                'incoming_agricultural', 'farm_expenses', 'lasting_investment', 'no_lasting_investmen']
# L2正则化特征柱状图
COLUMNS3 = ['Ville_id', 'Married', 'Number_children', 'education_level', 'total_members', 'incoming_salary',
            'incoming_own_farm', 'incoming_business', 'incoming_no_business', 'labor_primary', 'sex_0', 'sex_1',
            'Age', 'gained_asset', 'durable_asset', 'save_asset', 'living_expenses', 'other_expenses',
            'incoming_agricultural', 'farm_expenses', 'lasting_investment', 'no_lasting_investmen']


class Depression:

    def __init__(self, data):
        self.data = data
        # 产看是否导入成功
        # print(self.data.info())

        # 保存文件时的变量
        self.i = 0
        self.j = 0
        self.k = 0
        self.m = 0

    # 数据预处理
    def preprocess(self):
        # 查看有空值的列
        # missing_values_count = self.data.isnull().sum()
        # print(missing_values_count)

        # 查看过后发现只有no_lasting_investmen列中有20条缺失数据
        # 除去有缺失值的的样本
        # self.data.dropna(axis=0, inplace=True)
        # 查看是否删除成功
        # print(self.data.info())

        # 对数据进行按照index_col排序
        self.data = self.data.sort_values(by="Survey_id")
        # 或者用中位数进行填充
        self.data.loc[:, "no_lasting_investmen"] = self.data.loc[:, "no_lasting_investmen"].fillna(
            self.data.loc[:, "no_lasting_investmen"].median())
        # 查看是否填充成功
        # print(self.data.info())

        return self.data

    def preprocessDetail(self):
        # 特征类别都为数字(int 和 float)不需要进行特征值转换为数字特征否则要进行转换
        # 特征类属于分类型的只有性别和城市(所以为了数据的准确性需要对性别进行独热编码配置)
        # 如果不进行哑变量的处理就会把分类的数据训练成连续可计算的数据
        encode_data = self.oneHotEncode()
        # print(encode_data.head())

        # 无量纲化: 对数据进行标准化, 降低数据的误差
        standard_data = self.standardData(encode_data)
        # print(standard_data)

        # 对连续性数据进行分箱，防止过拟合
        discretizer_data = self.binsDiscretizer(standard_data)
        # print(discretizer_data.head())

        return discretizer_data

    # 独热编码处理独立不相关的特征
    def oneHotEncode(self):
        enc = OneHotEncoder(categories='auto').fit(self.data.iloc[:, 1:2])
        result = enc.transform(self.data.iloc[:, 1:2]).toarray()
        # 查看独热编码做哑变量特征的命名
        print("哑变量特征的命名: %s" % enc.get_feature_names_out())
        dic = {}
        for i in range(len(self.data)):
            dic[i] = i + 1
        # 对独热编码做哑变量特征进行跨行合并,左右横向合并
        df = pd.DataFrame(result)
        df = df.rename(dic)
        new_data = pd.concat([self.data, df], axis=1)
        # 删除性别和城市特征, 独热编码做哑变量特征进行命名
        new_data.drop(COLUMNS1_1, axis=1, inplace=True)
        new_data.columns = COLUMNS1
        # 将结果移到最后一列
        depressed = new_data.pop('depressed')
        new_data.insert(loc=new_data.shape[1], column='depressed', value=depressed, allow_duplicates=False)
        return new_data

    # 对连续型变量进行分箱
    def binsDiscretizer(self, data):
        new_data = data.copy()
        # 获取出连续特征的列['Age', 'gained_asset', 'durable_asset', 'save_asset', 'living_expenses', 'other_expenses', 'incoming_agricultural', 'farm_expenses', 'lasting_investment', 'no_lasting_investmen']
        result = KBinsDiscretizer(n_bins=2, encode='ordinal', strategy='uniform').fit_transform(
            new_data.loc[:, COLUMNS2_2])
        dic = {}
        for i in range(len(data)):
            dic[i] = i + 1
        df = pd.DataFrame(result)
        df = df.rename(dic)
        new_data = pd.concat([new_data, df], axis=1)
        # 删除性别和城市特征, 独热编码做哑变量特征进行命名
        new_data.drop(COLUMNS2_2, axis=1, inplace=True)
        new_data.columns = COLUMNS2
        # 将结果移到最后一列
        depressed = new_data.pop('depressed')
        new_data.insert(loc=new_data.shape[1], column='depressed', value=depressed, allow_duplicates=False)
        return new_data

    # 对数据进行标准化
    def standardData(self, data):
        scalar = StandardScaler()
        new_data = data.copy()
        depressed = new_data.pop('depressed')
        new_data_frame = scalar.fit_transform(new_data)
        new_data = pd.DataFrame(new_data_frame, index=new_data.index, columns=new_data.columns)
        new_data.insert(loc=new_data.shape[1], column='depressed', value=depressed, allow_duplicates=False)
        return new_data

    # 决策树
    def decisionTree(self, Xtrain, Xtest, ytrain, ytest):
        max_depthl.clear()
        max_featuresl.clear()
        decision_tree_classifier = tree.DecisionTreeClassifier()
        # KFold交叉采样(根据n_splits分成n个互斥子集, 每次使用一个作为测试集, n-1个作为训练集)
        cross_validation = StratifiedKFold(n_splits=10).get_n_splits(ytrain)
        # 网格搜索进行调参
        gridsearch = GridSearchCV(decision_tree_classifier,
                                  param_grid=parameter_grid,
                                  cv=cross_validation)
        # 得分最高的参数值，并构建最佳的决策树
        gridsearch.fit(Xtrain, ytrain)
        best_param = gridsearch.best_params_
        max_depthl.append(best_param['max_depth'])
        max_featuresl.append(best_param['max_features'])
        print("决策树的最优参数为max_depth = %s, max_feature = %s" % (best_param['max_depth'], best_param['max_features']))
        # 建立决策树模型, 随机参数为10, 随机选项为random,分支最大样本数为5, max_depth为树的深度, max_features限定样本个数
        clf = tree.DecisionTreeClassifier(criterion="entropy", random_state=90, splitter="random"
                                          , max_depth=best_param['max_depth']
                                          , max_features=best_param['max_features'])
        clf = clf.fit(Xtrain, ytrain)
        # 模型准确度
        score = clf.score(Xtest, ytest)
        # 计算均方误差(预测值与真实值之差的平方和的平均值)和均方根误差( f(x) 与样本真实值 y 之间距离平方的平均值，取结果后再开方)
        data_train_predictions = clf.predict(Xtest)
        lin_mse = mean_squared_error(ytest, data_train_predictions)
        lin_rmse = np.sqrt(lin_mse)  # 训练集上的RMSE
        # 计算精度和召回率
        y_train_pred = cross_val_predict(clf, Xtrain, ytrain, cv=10)
        precision = precision_score(ytrain, y_train_pred)
        recall = recall_score(ytrain, y_train_pred)
        print("决策树模型的均方误差(mse)为%s, 均方根误差(rmse)为%s" % (lin_mse, lin_rmse))
        print("决策树模型的精度(precision)为%s, 召回率(recall)为%s" % (precision, recall))
        print("决策树模型的预测精准度为%s" % score)
        # 将决策树可视化
        dot_data = tree.export_graphviz(clf, feature_names=feature_name, class_names=["depressed", "undepressed"],
                                        filled=True, rounded=True)
        graph = graphviz.Source(dot_data)
        graph.view()

        return score

    # 随机森林
    def randomForest(self, Xtrain, Xtest, ytrain, ytest):
        random_forest_classifier = RandomForestClassifier()
        cross_validation = StratifiedKFold(n_splits=10).get_n_splits(ytrain.ravel())
        gridsearch = GridSearchCV(random_forest_classifier,
                                  param_grid=parameter_grid,
                                  cv=cross_validation)
        gridsearch.fit(Xtrain.values.tolist(), ytrain.ravel())
        best_param = gridsearch.best_params_
        max_depthl.append(best_param['max_depth'])
        max_featuresl.append(best_param['max_features'])

        plt.figure()
        plt.bar(range(len(max_depthl)), max_depthl,
                tick_label=['df', 'rf'])
        plt.ylabel('max_depth')
        plt.xlabel('type(1:df,2:rf)')
        plt.title('best max_depth')

        self.i = self.i + 1
        # plt.savefig('D:\\desktop\\大作业截图\\10组决策树与随机森林剪枝参数网格搜索最优max_depth对比\\max_depth' + str(self.i) +'.jpg')
        plt.show()

        plt.figure()
        plt.bar(range(len(max_featuresl)), max_featuresl,
                tick_label=['df', 'rf'])
        plt.ylabel('max_featuresl')
        plt.xlabel('type(1:df,2:rf)')
        plt.title('best max_features')
        self.j = self.j + 1
        # plt.savefig('D:\\desktop\\大作业截图\\10组决策树与随机森林剪枝参数网格搜索最优max_features对比\\max_features' + str(self.j) +'.jpg')
        plt.show()

        print("随机森林的最优参数为max_depth = %s, max_feature = %s" % (best_param['max_depth'], best_param['max_features']))
        rfc = RandomForestClassifier(n_estimators=112, random_state=90
                                     , max_depth=best_param['max_depth']
                                     , max_features=best_param['max_features'])
        rfc = rfc.fit(Xtrain.values.tolist(), ytrain.ravel())
        # 模型准确度
        score = rfc.score(Xtest.values, ytest.values)
        # 计算均方误差(预测值与真实值之差的平方和的平均值)和均方根误差( f(x) 与样本真实值 y 之间距离平方的平均值，取结果后再开方)
        data_train_predictions = rfc.predict(Xtest.values)
        lin_mse = mean_squared_error(ytest.values, data_train_predictions)
        lin_rmse = np.sqrt(lin_mse)  # 训练集上的RMSE
        # 计算精度和召回率
        y_train_pred = cross_val_predict(rfc, Xtrain.values, ytrain.values, cv=10)
        precision = precision_score(ytrain.values, y_train_pred)
        recall = recall_score(ytrain.values, y_train_pred)
        print("随机森林模型的均方误差(mse)为%s, 均方根误差(rmse)为%s" % (lin_mse, lin_rmse))
        print("随机森林模型的精度(precision)为%s, 召回率(recall)为%s" % (precision, recall))
        print("随机森林模型的预测精准度为%s" % score)

        return score

    # 随机森林调参(使用学习曲线)
    def fitEstimators(self, X, y):
        # 确定范围
        scorel = []
        for i in range(0, 200, 10):
            rfc = RandomForestClassifier(n_estimators=i + 1, n_jobs=-1, random_state=90)
            score = cross_val_score(rfc, X, y, cv=10).mean()
            scorel.append(score)
        # 打印最大值及最大值对应的index
        print("第一次查找随机森林的estimators得分: %s, 最优值: %s " % (max(scorel), (scorel.index(max(scorel)) * 10 + 1)))
        plt.figure(figsize=[20, 5])
        plt.plot(range(1, 201, 10), scorel)
        # plt.savefig('D:\\desktop\\大作业截图\\第一次查找随机森林estimators参数.jpg')
        plt.show()
        # 范围索引(0.9020289569585345 121)
        best_index = scorel.index(max(scorel)) * 10 + 1
        # 进一步查找
        scorel = []
        for i in range(best_index - 10, best_index + 10):
            rfc = RandomForestClassifier(n_estimators=i, n_jobs=-1, random_state=90)
            score = cross_val_score(rfc, X, y, cv=10).mean()
            scorel.append(score)
        # 打印最大值及最大值对应的index(0.902728257657835 112)
        print("第二次查找随机森林的estimators得分: %s, 最优值: %s " % (
            max(scorel), ([*range(best_index - 10, best_index + 10)][scorel.index(max(scorel))])))
        plt.figure(figsize=[20, 5])
        plt.plot(range(best_index - 10, best_index + 10), scorel)
        # plt.savefig('D:\\desktop\\大作业截图\\第二次查找随机森林estimators参数.jpg')
        plt.show()

    # 逻辑回归
    def lgPredict(self, Xtrain, Xtest, ytrain, ytest):
        LR_ = LR(penalty="l2", solver="liblinear", random_state=420, max_iter=1000)
        LR_.fit(Xtrain, ytrain)
        # 打印每个特征的权重
        print("逻辑回归L2正则化每个特征的权重: %s" % LR_.coef_)
        # 可视化特征的权重
        plt.figure(figsize=[60, 5])
        plt.bar(range(len(LR_.coef_[0])), abs(LR_.coef_[0]),
                color=['blue' if item > 0 else 'red' for item in LR_.coef_[0]],
                tick_label=COLUMNS3)
        plt.xlabel('characters')
        plt.ylabel('weight')
        plt.title('weight of characters(red is negative)')
        self.k = self.k + 1
        # plt.savefig('D:\\desktop\\大作业截图\\10组逻辑回归L2正则化特征权重\\weight' + str(self.k) +'.jpg')
        plt.show()
        # 预测准确率
        score = LR_.score(Xtest, ytest)
        # 计算均方误差(预测值与真实值之差的平方和的平均值)和均方根误差( f(x) 与样本真实值 y 之间距离平方的平均值，取结果后再开方)
        data_train_predictions = LR_.predict(Xtest)
        lin_mse = mean_squared_error(ytest, data_train_predictions)
        lin_rmse = np.sqrt(lin_mse)  # 训练集上的RMSE
        # 计算精度和召回率
        y_train_pred = cross_val_predict(LR_, Xtrain, ytrain, cv=10)
        precision = precision_score(ytrain, y_train_pred)
        recall = recall_score(ytrain, y_train_pred)
        print("逻辑回归模型的均方误差(mse)为%s, 均方根误差(rmse)为%s" % (lin_mse, lin_rmse))
        print("逻辑回归模型的精度(precision)为%s, 召回率(recall)为%s" % (precision, recall))
        print("逻辑回归模型的预测精准度为%s" % score)
        return score

    # SVM支持向量机预测
    def svmPredict(self, Xtrain, Xtest, ytrain, ytest):
        kernels = ["linear", "poly", "rbf", "sigmoid"]
        scores = []
        difference = []
        for kernel in kernels:
            time0 = time()
            clf = SVC(kernel=kernel, gamma="auto", cache_size=2000).fit(Xtrain, ytrain)
            score = clf.score(Xtest, ytest)
            run_time = time() - time0
            scores.append(score)
            print("在kernel为%s时, 准确率为%f" % (kernel, score))
            print("运行时间为%s" % (datetime.datetime.fromtimestamp(run_time).strftime("%M:%S:%f")))
        selected_kernel = kernels[scores.index(max(scores))]
        print("选取的kernel值为%s" % selected_kernel)
        for item in scores:
            difference.append(item - np.floor(min(scores) * 10) / 10)
        plt.figure(figsize=[20, 10])
        plt.bar(range(len(scores)), difference,
                tick_label=kernels)
        plt.xlabel('Kernel')
        plt.ylabel('Accuracy - float(np.floor(min(scores) * 10) / 10)')
        plt.title('select kernel')
        self.m = self.m + 1
        # plt.savefig('D:\\desktop\\大作业截图\\10组svm核函数选择\\selectedKernel' + str(self.m) +'.jpg')
        max_score = max(scores)
        clf = SVC(kernel=selected_kernel, gamma="auto", cache_size=2000).fit(Xtrain, ytrain)
        data_train_predictions = clf.predict(Xtest)
        lin_mse = mean_squared_error(ytest, data_train_predictions)
        lin_rmse = np.sqrt(lin_mse)  # 训练集上的RMSE
        y_train_pred = cross_val_predict(clf, Xtrain, ytrain, cv=10)
        precision = precision_score(ytrain, y_train_pred)
        recall = recall_score(ytrain, y_train_pred)
        print("svm模型的均方误差(mse)为%s, 均方根误差(rmse)为%s" % (lin_mse, lin_rmse))
        print("svm模型的精度(precision)为%s, 召回率(recall)为%s" % (precision, recall))
        print("svm模型的预测精准度为%s" % max_score)
        return max_score


if __name__ == '__main__':
    # 读取数据
    data = pd.read_csv('data/b_depressed.csv', index_col=0)
    # print(data)

    # 保护data原本的数据不被覆盖
    data_ = data.copy()
    depression = Depression(data_)

    # 进行训练次数
    size1 = 10
    size2 = 10000
    # 决策树准确率集
    df_l = []
    # 随机森林准确率集
    rf_l = []
    # 逻辑回归准确率集
    lg_l = []
    # svm支持向量机预测
    svm_l = []

    # 对逻辑回归进行预处理
    depression.preprocess();
    real_data = depression.preprocessDetail()
    X = real_data.iloc[:, :-1]
    y = real_data.iloc[:, -1]
    # 测试随机森林n_estimators的最优值(112)
    depression.fitEstimators(X.values.tolist(), y.ravel())

    for i in range(1, size1 + 1):
        # 区分测试集和训练集
        Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.3)

        # 建立决策树模型进行预测
        score_1 = depression.decisionTree(Xtrain, Xtest, ytrain, ytest)
        df_l.append(score_1)

        # 随机森林模型进行预测
        score_2 = depression.randomForest(Xtrain, Xtest, ytrain, ytest)
        rf_l.append(score_2)

        # 逻辑回归模型预测
        score_3 = depression.lgPredict(Xtrain, Xtest, ytrain, ytest)
        lg_l.append(score_3)

        # svm支持向量机预测
        score_4 = depression.svmPredict(Xtrain, Xtest, ytrain, ytest)
        svm_l.append(score_4)

        print("-------------------第%d组结束-------------------" % i)

    # Logistic、决策树、随机森林和svm性能评估
    print("Logistic回归预测准确率最大值为: %f, 最小值为: %f." % (max(lg_l), min(lg_l)))
    print("决策树预测准确率最大值为: %f, 最小值为: %f." % (max(df_l), min(df_l)))
    print("随机森林预测准确率最大值为: %f, 最小值为: %f." % (max(rf_l), min(rf_l)))
    print("svm预测准确率最大值为: %f, 最小值为: %f." % (max(svm_l), min(svm_l)))
    # 决策树和随机森林预测可视化
    plt.figure(figsize=[20, 5])
    plt.plot(range(1, size1 + 1, 1), df_l, label="DecisionTree")
    plt.plot(range(1, size1 + 1, 1), rf_l, label="RandomForest")
    plt.plot(range(1, size1 + 1, 1), lg_l, label="Logistic")
    plt.plot(range(1, size1 + 1, 1), svm_l, label="SVM")
    plt.legend()
    # plt.savefig('D:\\desktop\\大作业截图\\10组4个模型预测准确率对比.jpg')
    plt.show()