机器学习分类通用代码

本文主要提供一种通用的机器学习分类代码，通过重写两个函数完成处理和评价。
提供两种主函数代码，一个用于常规单模型训练和保存，一个用于选取最优模型并生成对比模型数据表格。

主要步骤为：

  • 重写数据预处理函数（本文以文本数据TF-IDF为例子）
  • 重写评价函数（根据需求从metrics中选取，文本以准去率、精确率、召回率、F1值作为评价标准）
  • 载入主函数（数据尽量使用pandas读取结构化类型数据为主，可以先用选优模型主函数调参选取最优模型，再使用单模型调最优模型）

后续版本将要更新的内容：

  • 增加预测函数 （读取保存的模型和预处理配置，预测单条数据或多条数据）
  • 增加分类结果可视化
  • 增加不同的预处理方式样例（增加图片预处理、增加表格数据预处理、增加时序数据预处理）
  • 增加不同的评测方法函数

    导包

    ```python 导包 import json import time import joblib import pickle import jieba import pandas as pd

    sklearn

    from sklearn.preprocessing import LabelEncoder from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import RidgeClassifier,LogisticRegression from sklearn.naive_bayes import MultinomialNB,BernoulliNB,ComplementNB from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier from sklearn.svm import SVC from sklearn.tree import DecisionTreeClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import precision_score,recall_score,f1_score,accuracy_score from lightgbm import LGBMClassifier import warnings warnings.filterwarnings(“ignore”, category=DeprecationWarning) warnings.simplefilter(“ignore”)
  2. <a name="AuGXF"></a>
  3. ## 数据预处理函数
  4. ```python
  5. # 以文本数据为例子,文本数据预处理通常需要使用某种技术转化为稠密向量
  6. # 以TF_IDF为例子
  7. def TF_IDF_Deal(train_data,content_col,label_col):
  8.     # 数据预处理部分
  9.     train_data.drop(0, inplace=True, axis=0)
  10.     train_data = train_data[[content_col,label_col]]
  11.     # sample(表示随机抽取 frac表示比例)
  12.     # train_data = train_data.sample(frac=1.0)
  13.     # 标签编码并保存
  14.     target_names = train_data[label_col].value_counts().index
  15.     lbl = LabelEncoder().fit(target_names)
  16.     train_data[label_col] = lbl.transform(train_data[label_col])
  17.     name_label = {i:str(j) for i, j in zip(lbl.classes_, lbl.transform(target_names))}
  18.     label_name = {str(j):i for i, j in zip(lbl.classes_, lbl.transform(target_names))}
  19.     # target_names = dict(sorted(key_value.items(), key=lambda kv: (kv[1], kv[0])))
  20.     with open("label_name.json","w") as f:
  21.         json.dump(label_name,f)
  22.     with open("name_label.json","w") as f:
  23.         json.dump(name_label,f)
  24.     # 仅对标题做分词
  25.     def strcut(s):
  26.         seg_list = jieba.cut(s)
  27.         return ' '.join(list(seg_list))
  29.     train_title = train_data[content_col].apply(strcut)
  30.     # TF-IDF
  31.     tfidf = TfidfVectorizer(ngram_range=(1, 1))
  32.     tfidf.fit_transform(train_title)
  33.     train_title_ttidf = tfidf.fit_transform(train_title)
  34.     with open("savemodel/TF_IDF.pkl","wb") as f:
  35.         pickle.dump(tfidf,f)
  36.     # 分割数据集
  37.     tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = train_test_split(
  38.         train_data[content_col], train_title_ttidf, train_data[label_col],
  39.         stratify=train_data[label_col],
  40.         # shuffle=True,
  41.         test_size=0.2,
  42.         random_state=666
  43.     )
  45.     return tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y

评测函数

  1. def predict_metrics_PRF(clf,clf_name,val_tfidf,val_y,with_return=False):
  3.     val_pred = clf.predict(val_tfidf)
  4.     accuracy = accuracy_score(y_true=val_y,y_pred=val_pred)
  5.     precision = precision_score(y_true=val_y,y_pred=val_pred,average='macro')
  6.     recall = recall_score(y_true=val_y,y_pred=val_pred,average='macro')
  7.     f1score = f1_score(y_true=val_y,y_pred=val_pred,average='macro')
  8.     if with_return:
  9.         return accuracy,precision,recall,f1score
  10.     else:
  11.         print("{}:".format(clf_name))
  12.         print("accuracy: {:.3f}\nprecision: {:.3f}".format(accuracy, precision))
  13.         print("recall: {:.3f}\nf1score: {:.3f}".format(recall, f1score))

主函数

根据自己数据类型和数据处理方式的不同重写 数据预处理函数
根据自己所需的评价指标重写 评测函数
重写上述两个函数后，可以通过以下结构的代码进行调用，仅用7行代码构成主函数完成一个对机器学习分类模型的训练和保存 

  1. if __name__ == '__main__':
  2.     # 读取数据
  3.     train_data = pd.read_excel('../data/3_训练数据集20210414_A1.xlsx')
  4.     # 数据预处理函数
  5.     tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = TF_IDF_Deal(train_data, 'NEWS_TITLE', 'LABEL')
  6.     # 模型建立、模型训练、模型保存
  7.     clf_name = "RidgeClassifier"
  8.     clf = RidgeClassifier()
  9.     clf.fit(tr_tfidf, tr_y)  # 训练
  10.     joblib.dump(clf, "savemodel/{}.model".format(clf_name))  # 保存模型
  11.     # 评测函数
  12.     predict_metrics_PRF(clf, clf_name,val_tfidf, val_y)

选优模型代码

考虑到不管是比赛还是写paper都需要模型对比，故提供一种选优模型代码，通过遍历所有sklearn库中的分类方法来查看最优模型。
注意：

  • 原始代码中模型API都为默认参数，可自己进行调参
  • 某些模型必须使用稠密向量，故没有加入进入，如LDA判别模型等
    1. if __name__ == '__main__':
    2.   # company_name = pd.read_excel('../data/2_企业公司名.xlsx', names=['name'])
    3.   train_data = pd.read_excel('../data/3_训练数据集20210414_A1.xlsx')
    4.   # 数据预处理
    5.   tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = TF_IDF_Deal(train_data,'NEWS_TITLE','LABEL')
    6.   # model compare
    7.   model_names = ['RidgeClassifier','LogisticRegression','SGDClassifier', # 线性模型
    8.                  'MultinomialNB','BernoulliNB','ComplementNB', # 贝叶斯模型  未加入GaussianNB
    9.                  'KNeighborsClassifier',  # 紧邻模型
    10.                  'RandomForestClassifier','AdaBoostClassifier','GradientBoostingClassifier', # 集成模型
    11.                  'SVC', 'DecisionTreeClassifier', # #支持向量机、决策树模型
    12.                  'LGBMClassifier',                # LGB  kaggle三大杀器之一
    13.                  ]
    14.   models = [RidgeClassifier(),
    15.             LogisticRegression(),
    16.             SGDClassifier(),
    17.             MultinomialNB(),
    18.             BernoulliNB(),
    19.             ComplementNB(),
    20.             KNeighborsClassifier(),
    21.             RandomForestClassifier(),
    22.             AdaBoostClassifier(),
    23.             GradientBoostingClassifier(),
    24.             SVC(),
    25.             DecisionTreeClassifier(),
    26.             LGBMClassifier()
    27.             ]
    28.   # 找到最优模型
    29.   all_time = time.time() # 记录开始时间
    30.   metrics_df = pd.DataFrame([],columns=["model","accuracy","precision","recall","f1_score"]) # 存表
    31.   for idx,(model,model_name) in enumerate(zip(models,model_names)):
    32.       start = time.time()
    33.       clf = model.fit(tr_tfidf,tr_y)
    34.       # 如果重写了评测函数 需要修改这边的返回值 以及更改生成列表的columns名
    35.       accuracy,precision,recall,f1score = predict_metrics_PRF(clf,model_name,val_tfidf,val_y,with_return=True)
    36.       metrics_df.loc[len(metrics_df)] = [model_name,accuracy,precision,recall,f1score]
    37.       print("{}/{} {} 耗时： {:.3f}sec".format(idx+1,len(model_names),model_name,time.time()-start)) # 打印进度
    38.   print("总耗时: {:.3f}sec".format(time.time()-all_time))
    39.   print(metrics_df)
    40.   metrics_df.to_csv("best_model_metrics.csv",index=False)

效果
image.png

完整代码

  1. #!/usr/bin/env python
  2. # -*- encoding: utf-8 -*-
  3. '''
  4. @File    :   TF_IDF_Deal.py
  5. @Contact :   htkstudy@163.com
  7. @Modify Time      @Author    @Version    @Desciption
  8. ------------      -------    --------    -----------
  9. 2021/5/7 18:29   Armor(htk)     1.0         None
  10. '''
  12. import json
  13. import time
  14. import joblib
  15. import pickle
  16. import jieba
  17. import pandas as pd
  19. from sklearn.preprocessing import LabelEncoder
  20. from sklearn.svm import SVC
  21. from sklearn.tree import DecisionTreeClassifier
  22. from sklearn.neighbors import KNeighborsClassifier
  23. from sklearn.feature_extraction.text import TfidfVectorizer
  24. from sklearn.linear_model import RidgeClassifier,LogisticRegression,SGDClassifier
  25. from sklearn.naive_bayes import MultinomialNB,BernoulliNB,ComplementNB,GaussianNB
  26. from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier,GradientBoostingClassifier
  27. from sklearn.discriminant_analysis import LinearDiscriminantAnalysis,QuadraticDiscriminantAnalysis
  29. from sklearn.model_selection import train_test_split
  30. from sklearn.metrics import precision_score,recall_score,f1_score,accuracy_score
  32. from lightgbm import LGBMClassifier
  33. import warnings
  34. warnings.filterwarnings("ignore", category=DeprecationWarning)
  35. warnings.simplefilter("ignore")
  37. def TF_IDF_Deal(train_data,content_col,label_col):
  38.     # 数据预处理部分
  39.     train_data.drop(0, inplace=True, axis=0)
  40.     train_data = train_data[[content_col,label_col]]
  41.     # sample(表示随机抽取 frac表示比例)
  42.     # train_data = train_data.sample(frac=1.0)
  43.     # 标签编码并保存
  44.     target_names = train_data[label_col].value_counts().index
  45.     lbl = LabelEncoder().fit(target_names)
  46.     train_data[label_col] = lbl.transform(train_data[label_col])
  47.     name_label = {i:str(j) for i, j in zip(lbl.classes_, lbl.transform(target_names))}
  48.     label_name = {str(j):i for i, j in zip(lbl.classes_, lbl.transform(target_names))}
  49.     # target_names = dict(sorted(key_value.items(), key=lambda kv: (kv[1], kv[0])))
  50.     with open("label_name.json","w") as f:
  51.         json.dump(label_name,f)
  52.     with open("name_label.json","w") as f:
  53.         json.dump(name_label,f)
  54.     # 仅对标题做分词
  55.     def strcut(s):
  56.         seg_list = jieba.cut(s)
  57.         return ' '.join(list(seg_list))
  59.     train_title = train_data[content_col].apply(strcut)
  60.     # TF-IDF
  61.     tfidf = TfidfVectorizer(ngram_range=(1, 1))
  62.     tfidf.fit_transform(train_title)
  63.     train_title_ttidf = tfidf.fit_transform(train_title)
  64.     with open("savemodel/TF_IDF.pkl","wb") as f:
  65.         pickle.dump(tfidf,f)
  66.     # 分割数据集
  67.     tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = train_test_split(
  68.         train_data[content_col], train_title_ttidf, train_data[label_col],
  69.         stratify=train_data[label_col],
  70.         # shuffle=True,
  71.         test_size=0.2,
  72.         random_state=666
  73.     )
  75.     return tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y
  77. def predict_metrics_PRF(clf,clf_name,val_tfidf,val_y,with_return=False):
  79.     val_pred = clf.predict(val_tfidf)
  80.     val_true = val_y.values.tolist()
  81.     accuracy = accuracy_score(y_true=val_true,y_pred=val_pred)
  82.     precision = precision_score(y_true=val_true,y_pred=val_pred,average='macro')
  83.     recall = recall_score(y_true=val_true,y_pred=val_pred,average='macro')
  84.     f1score = f1_score(y_true=val_true,y_pred=val_pred,average='macro')
  85.     if with_return:
  86.         return accuracy,precision,recall,f1score
  87.     else:
  88.         print("{}:".format(clf_name))
  89.         print("accuracy: {:.3f}\nprecision: {:.3f}".format(accuracy, precision))
  90.         print("recall: {:.3f}\nf1score: {:.3f}".format(recall, f1score))
  92. def model_example():
  93.     # company_name = pd.read_excel('../data/2_企业公司名.xlsx', names=['name'])
  94.     train_data = pd.read_excel('../data/3_训练数据集20210414_A1.xlsx')
  95.     # 数据预处理
  96.     tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = TF_IDF_Deal(train_data, 'NEWS_TITLE', 'LABEL')
  97.     # 模型构造
  98.     clf_name = "RidgeClassifier"
  99.     clf = RidgeClassifier()
  100.     clf.fit(tr_tfidf, tr_y)  # 训练
  101.     joblib.dump(clf, "savemodel/{}.model".format(clf_name))  # 保存模型
  102.     # 评价效果
  103.     predict_metrics_PRF(clf, clf_name,val_tfidf, val_y)
  106. # if __name__ == '__main__':
  107. #     # 数据预处理
  108. #     with open("savemodel/TF_IDF.pkl",'rb') as f:
  109. #         tfidf = pickle.load(f)
  110. #     val_x = tfidf.fit_transform(val_x)
  111. #     print(val_x)
  112. #     # 加载模型
  113. #     clf = joblib.load(savemodel_path)
  114. #     # 预测
  115. #     val_pred = clf.predict(val_x)
  116. #     with open("label_name.json",'r') as f:
  117. #         key_values = json.load(f)
  118. #     label_pred = [key_values[str(i)] for i in val_pred]
  119. #     return val_pred,label_pred
  121. if __name__ == '__main__':
  122.     # company_name = pd.read_excel('../data/2_企业公司名.xlsx', names=['name'])
  123.     train_data = pd.read_excel('../data/3_训练数据集20210414_A1.xlsx')
  124.     # 数据预处理
  125.     tr_x, val_x, tr_tfidf, val_tfidf, tr_y, val_y = TF_IDF_Deal(train_data,'NEWS_TITLE','LABEL')
  126.     # model compare
  127.     model_names = ['RidgeClassifier','LogisticRegression','SGDClassifier', # 线性模型
  128.                    'MultinomialNB','BernoulliNB','ComplementNB', # 贝叶斯模型  未加入GaussianNB
  129.                    'KNeighborsClassifier',  # 紧邻模型
  130.                    'RandomForestClassifier','AdaBoostClassifier','GradientBoostingClassifier', # 集成模型
  131.                    'SVC', 'DecisionTreeClassifier', # #支持向量机、决策树模型
  132.                    'LGBMClassifier',                # LGB  kaggle三大杀器之一
  133.                    ]
  134.     models = [RidgeClassifier(),
  135.               LogisticRegression(),
  136.               SGDClassifier(),
  137.               MultinomialNB(),
  138.               BernoulliNB(),
  139.               ComplementNB(),
  140.               KNeighborsClassifier(),
  141.               RandomForestClassifier(),
  142.               AdaBoostClassifier(),
  143.               GradientBoostingClassifier(),
  144.               SVC(),
  145.               DecisionTreeClassifier(),
  146.               LGBMClassifier()
  147.               ]
  148.     # 找到最优模型
  149.     all_time = time.time() # 记录开始时间
  150.     metrics_df = pd.DataFrame([],columns=["model","accuracy","precision","recall","f1_score"]) # 存表
  151.     for idx,(model,model_name) in enumerate(zip(models,model_names)):
  152.         start = time.time()
  153.         clf = model.fit(tr_tfidf,tr_y)
  154.         # 如果重写了评测函数 需要修改这边的返回值 以及更改生成列表的columns名
  155.         accuracy,precision,recall,f1score = predict_metrics_PRF(clf,model_name,val_tfidf,val_y,with_return=True)
  156.         metrics_df.loc[len(metrics_df)] = [model_name,accuracy,precision,recall,f1score]
  157.         print("{}/{} {} 耗时： {:.3f}sec".format(idx+1,len(model_names),model_name,time.time()-start)) # 打印进度
  158.     print("总耗时: {:.3f}sec".format(time.time()-all_time))
  159.     print(metrics_df)
  160.     metrics_df.to_csv("best_model_metrics.csv",index=False)