数据描述

MNIST 数据集可在 http://yann.lecun.com/exdb/mnist/ 获取, 它包含了四个部分:

  • Training set images: train-images-idx3-ubyte.gz (9.9 MB, 解压后 47 MB, 包含 60,000 个样本)
  • Training set labels: train-labels-idx1-ubyte.gz (29 KB, 解压后 60 KB, 包含 60,000 个标签)
  • Test set images: t10k-images-idx3-ubyte.gz (1.6 MB, 解压后 7.8 MB, 包含 10,000 个样本)
  • Test set labels: t10k-labels-idx1-ubyte.gz (5KB, 解压后 10 KB, 包含 10,000 个标签)

MNIST 数据集来自美国国家标准与技术研究所, National Institute of Standards and Technology (NIST). 训练集 (training set) 由来自 250 个不同人手写的数字构成, 其中 50% 是高中学生, 50% 来自人口普查局 (the Census Bureau) 的工作人员. 测试集(test set) 也是同样比例的手写数字数据.

keras普通实现

  1. from keras.datasets import mnist
  2. from keras.models import Sequential
  3. from keras.layers import Dense
  5. import matplotlib.pyplot as plt
  7. from sklearn.preprocessing import OneHotEncoder
  9. def load_data():
  10.     (x_train, y_train), (x_test, y_test) = mnist.load_data()
  11.     x_train = x_train.reshape(x_train.shape[0], 28 * 28)
  12.     x_test = x_test.reshape(x_test.shape[0], 28 * 28)
  13.     y_train = OneHotEncoder().fit_transform(y_train.reshape(-1,1))
  14.     y_test = OneHotEncoder().fit_transform(y_test.reshape(-1,1))
  15.     return (x_train, y_train), (x_test, y_test)
  18. def build_model():
  19.     model = Sequential()
  20.     model.add(Dense(128, activation='relu', input_shape=(784, )))
  21.     model.add(Dense(64, activation='relu'))
  22.     model.add(Dense(32, activation='relu'))
  23.     model.add(Dense(16, activation='relu'))
  24.     model.add(Dense(10, activation='softmax'))
  25.     model.compile(optimizer='Adam',
  26.                   loss='categorical_crossentropy',
  27.                   metrics=['accuracy'])
  28.     return model
  30. #画出训练过程中训练和验证的精度与损失
  31. def draw_train_history(history):
  32.     plt.figure(1)
  34.     # summarize history for accuracy
  35.     plt.subplot(211)
  36.     plt.plot(history.history['accuracy'])
  37.     plt.plot(history.history['val_accuracy'])
  38.     plt.title('model accuracy')
  39.     plt.ylabel('accuracy')
  40.     plt.xlabel('epoch')
  41.     plt.legend(['train', 'validation'])
  43.     # summarize history for loss
  44.     plt.subplot(212)
  45.     plt.plot(history.history['loss'])
  46.     plt.plot(history.history['val_loss'])
  47.     plt.title('model loss')
  48.     plt.ylabel('loss')
  49.     plt.xlabel('epoch')
  50.     plt.legend(['train', 'validation'])
  51.     plt.show()
  54. if __name__ == '__main__':
  55.     (x_train, y_train), (x_test, y_test) = load_data()
  56.     model = build_model()
  57.     history = model.fit(x_train, y_train,
  58.                         epochs=20,
  59.                         batch_size=64,
  60.                         validation_split=0.3)
  61.     draw_train_history(history)
  62.     model.save("classification.h5")
  64.     loss, accuracy = model.evaluate(x_test, y_test)
  65.     print("test loss: {}, test accuracy: {}".format(loss, accuracy))
  67.     weights = model.get_weights()
  68.     print("weights: ", weights)

模型结构

分类-keras - 图1

模型输出

  1. test loss: 0.13618469612854534, test accuracy: 0.972000002861023

模型损失以及准确率曲线

分类-keras - 图2

keras-scikit_learn集成学习

  1. import numpy as np
  2. import matplotlib.pyplot as plt
  4. from keras.datasets import mnist
  5. from keras.wrappers.scikit_learn import KerasClassifier
  6. from keras.models import Sequential
  7. from keras.layers import Dense
  9. from sklearn.preprocessing import LabelEncoder
  10. from sklearn.ensemble import VotingClassifier
  11. from sklearn.model_selection import cross_val_score
  13. from pathlib import Path
  15. def load_data():
  16.     (x_train, y_train), (x_test, y_test) = mnist.load_data()
  17.     x_train = x_train.reshape(x_train.shape[0], 28 * 28)
  18.     x_test = x_test.reshape(x_test.shape[0], 28 * 28)
  19.     y_train = LabelEncoder().fit_transform(y_train.reshape(-1,1))
  20.     y_test = LabelEncoder().fit_transform(y_test.reshape(-1,1))
  21.     return (x_train, y_train), (x_test, y_test)
  24. #画出训练过程中训练和验证的精度与损失
  25. def draw_train_history(history):
  26.     plt.figure(1)
  28.     # summarize history for accuracy
  29.     plt.subplot(211)
  30.     plt.plot(history.history['accuracy'])
  31.     plt.plot(history.history['val_accuracy'])
  32.     plt.title('model accuracy')
  33.     plt.ylabel('accuracy')
  34.     plt.xlabel('epoch')
  35.     plt.legend(['train', 'validation'])
  37.     # summarize history for loss
  38.     plt.subplot(212)
  39.     plt.plot(history.history['loss'])
  40.     plt.plot(history.history['val_loss'])
  41.     plt.title('model loss')
  42.     plt.ylabel('loss')
  43.     plt.xlabel('epoch')
  44.     plt.legend(['train', 'validation'])
  45.     plt.show()
  48. def build_model(hidden_units):
  49.     model = Sequential()
  50.     for index, unit in enumerate(hidden_units):
  51.         if index == 0:
  52.             model.add(Dense(unit, activation='relu', input_shape=(784, )))
  53.         else:
  54.             model.add(Dense(unit, activation='relu'))
  55.     model.add(Dense(10, activation='softmax'))
  56.     model.compile(optimizer='Adam',
  57.                   loss='categorical_crossentropy',
  58.                   metrics=['accuracy'])
  59.     return model
  62. def build_model1():
  63.     model = Sequential()
  64.     model.add(Dense(128, activation='relu', input_shape=(784, )))
  65.     model.add(Dense(64, activation='relu'))
  66.     model.add(Dense(10, activation='softmax'))
  67.     model.compile(optimizer='Adam',
  68.                   loss='categorical_crossentropy',
  69.                   metrics=['accuracy'])
  70.     return model
  73. def build_model2():
  74.     model = Sequential()
  75.     model.add(Dense(64, activation='relu', input_shape=(784, )))
  76.     model.add(Dense(32, activation='relu'))
  77.     model.add(Dense(10, activation='softmax'))
  78.     model.compile(optimizer='Adam',
  79.                   loss='categorical_crossentropy',
  80.                   metrics=['accuracy'])
  81.     return model
  84. def build_model3():
  85.     model = Sequential()
  86.     model.add(Dense(32, activation='relu', input_shape=(784, )))
  87.     model.add(Dense(16, activation='relu'))
  88.     model.add(Dense(10, activation='softmax'))
  89.     model.compile(optimizer='Adam',
  90.                   loss='categorical_crossentropy',
  91.                   metrics=['accuracy'])
  92.     return model
  95. if __name__ == '__main__':
  96.     (x_train, y_train), (x_test, y_test) = load_data()
  98.     model1 = KerasClassifier(build_fn=build_model1, epochs=20, batch_size=64)
  99.     model1._estimator_type = "classifier"
  100.     model2 = KerasClassifier(build_fn=build_model2, epochs=20, batch_size=64)
  101.     model2._estimator_type = "classifier"
  102.     model3 = KerasClassifier(build_fn=build_model3, epochs=20, batch_size=64)
  103.     model3._estimator_type = "classifier"
  105.     # if ‘hard’, uses predicted class labels for majority rule voting.
  106.     # if ‘soft’, predicts the class label based on the argmax of the
  107.     # sums of the predicted probabilities,
  108.     # which is recommended for an ensemble of well-calibrated classifiers.
  109.     cls = VotingClassifier(estimators=(['model1', model1],
  110.                                        ['model2', model2],
  111.                                        ['model3', model3]),
  112.                            voting='hard')
  113.     cls.fit(x_train, y_train)
  115.     print("score: ", cls.score(x_test, y_test))

这里需要注意VotingClassifier中的voting参数：

  • hard：少数服从多数
  • soft：通过预测概率和的最大位置作为预测标签，也是比较推荐的方式

而且这里我们使用的是VotingClassifier，在scikit_learn中，也就代表将每个基学习器的结果通过不同的投票策略生成最终的输出，我们来看一下在sklearn文档中的描述

Soft Voting/Majority Rule classifier for unfitted estimators.

如果我们使用StackingClassifier的话，就是接下来要实现的学习法，我们来看一下sklearn文档中的描述

Stacked generalization consists in stacking the output of individual estimator and use a classifier to compute the final prediction. Stacking allows to use the strength of each individual estimator by using their output as input of a final estimator.

模型输出

  1. score:  0.9711

keras集成学习-学习法

  1. import matplotlib.pyplot as plt
  3. from keras.datasets import mnist
  4. from keras.models import Model
  5. from keras.layers import Input, Dense, concatenate
  7. from sklearn.preprocessing import OneHotEncoder
  9. def load_data():
  10.     (x_train, y_train), (x_test, y_test) = mnist.load_data()
  11.     x_train = x_train.reshape(x_train.shape[0], 28 * 28)
  12.     x_test = x_test.reshape(x_test.shape[0], 28 * 28)
  13.     y_train = OneHotEncoder().fit_transform(y_train.reshape(-1,1))
  14.     y_test = OneHotEncoder().fit_transform(y_test.reshape(-1,1))
  15.     return (x_train, y_train), (x_test, y_test)
  18. #画出训练过程中训练和验证的精度与损失
  19. def draw_train_history(history):
  20.     plt.figure(1)
  22.     # summarize history for accuracy
  23.     plt.subplot(211)
  24.     plt.plot(history.history['accuracy'])
  25.     plt.plot(history.history['val_accuracy'])
  26.     plt.title('model accuracy')
  27.     plt.ylabel('accuracy')
  28.     plt.xlabel('epoch')
  29.     plt.legend(['train', 'validation'])
  31.     # summarize history for loss
  32.     plt.subplot(212)
  33.     plt.plot(history.history['loss'])
  34.     plt.plot(history.history['val_loss'])
  35.     plt.title('model loss')
  36.     plt.ylabel('loss')
  37.     plt.xlabel('epoch')
  38.     plt.legend(['train', 'validation'])
  39.     plt.show()
  42. def build_model():
  43.     inputs = Input(shape=(784, ))
  44.     model1_1 = Dense(64, activation='relu')(inputs)
  45.     model2_1 = Dense(128, activation='relu')(inputs)
  46.     model3_1 = Dense(32, activation='relu')(inputs)
  47.     model1_2 = Dense(32, activation='relu')(model1_1)
  48.     model2_2 = Dense(64, activation='relu')(model2_1)
  49.     model3_2 = Dense(16, activation='relu')(model3_1)
  50.     model1_3 = Dense(16, activation='relu')(model1_2)
  51.     model2_3 = Dense(32, activation='relu')(model2_2)
  52.     model3_3 = Dense(8, activation='relu')(model3_2)
  53.     con = concatenate([model1_3, model2_3, model3_3])
  54.     output = Dense(10, activation='softmax')(con)
  55.     model = Model(inputs=inputs, outputs=output)
  56.     model.compile(optimizer='Adam',
  57.                   loss='categorical_crossentropy',
  58.                   metrics=['accuracy'])
  59.     return model
  62. if __name__ == '__main__':
  63.     (x_train, y_train), (x_test, y_test) = load_data()
  65.     model = build_model()
  66.     history = model.fit(x_train, y_train,
  67.                         epochs=20,
  68.                         batch_size=64,
  69.                         validation_split=0.3)
  70.     draw_train_history(history)
  71.     model.save("classification-learning-ensemble.h5")
  73.     loss, accuracy = model.evaluate(x_test, y_test)
  74.     print("test loss: {}, test accuracy: {}".format(loss, accuracy))

模型结构

分类-keras - 图3

模型输出

  1. test loss: 0.14912846596296658, test accuracy: 0.9682000279426575

模型损失以及准确率曲线

分类-keras - 图4

代码位置

https://github.com/Knowledge-Precipitation-Tribe/Neural-network/tree/master/code/Ensemble-Learning