疑问

  1. rule和glorot初始化不搭配吧？

    代码

    参考tensorflow-DeepFM/DeepFM.py ```python “”” Tensorflow implementation of DeepFM [1] Reference: [1] DeepFM: A Factorization-Machine based Neural Network for CTR Prediction, Huifeng Guo, Ruiming Tang, Yunming Yey, Zhenguo Li, Xiuqiang He. “””

import numpy as np import tensorflow as tf from sklearn.base import BaseEstimator, TransformerMixin from sklearn.metrics import roc_auc_score from time import time from tensorflow.contrib.layers.python.layers import batch_norm as batch_norm from yellowfin import YFOptimizer

class DeepFM(BaseEstimator, TransformerMixin): def init(self, feature_size, field_size, embedding_size=8, dropout_fm=[1.0, 1.0], deep_layers=[32, 32], dropout_deep=[0.5, 0.5, 0.5], deep_layers_activation=tf.nn.relu, epoch=10, batch_size=256, learning_rate=0.001, optimizer_type=”adam”, batch_norm=0, batch_norm_decay=0.995, verbose=False, random_seed=2016, use_fm=True, use_deep=True, loss_type=”logloss”, eval_metric=roc_auc_score, l2_reg=0.0, greater_is_better=True): assert (use_fm or use_deep) assert loss_type in [“logloss”, “mse”], \ “loss_type can be either ‘logloss’ for classification task or ‘mse’ for regression task”

  1.     self.feature_size = feature_size        # denote as M, size of the feature dictionary
  2.     self.field_size = field_size            # denote as F, size of the feature fields
  3.     self.embedding_size = embedding_size    # denote as K, size of the feature embedding
  5.     self.dropout_fm = dropout_fm
  6.     self.deep_layers = deep_layers
  7.     self.dropout_deep = dropout_deep
  8.     self.deep_layers_activation = deep_layers_activation
  9.     self.use_fm = use_fm
  10.     self.use_deep = use_deep
  11.     self.l2_reg = l2_reg
  13.     self.epoch = epoch
  14.     self.batch_size = batch_size
  15.     self.learning_rate = learning_rate
  16.     self.optimizer_type = optimizer_type
  18.     self.batch_norm = batch_norm
  19.     self.batch_norm_decay = batch_norm_decay
  21.     self.verbose = verbose
  22.     self.random_seed = random_seed
  23.     self.loss_type = loss_type
  24.     self.eval_metric = eval_metric
  25.     self.greater_is_better = greater_is_better
  26.     self.train_result, self.valid_result = [], []
  28.     self._init_graph()
  31. def _init_graph(self):
  32.     self.graph = tf.Graph()
  33.     with self.graph.as_default():
  35.         tf.set_random_seed(self.random_seed)
  37.         self.feat_index = tf.placeholder(tf.int32, shape=[None, None],
  38.                                              name="feat_index")  # None * F
  39.         self.feat_value = tf.placeholder(tf.float32, shape=[None, None],
  40.                                              name="feat_value")  # None * F
  41.         self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label")  # None * 1
  42.         self.dropout_keep_fm = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_fm")
  43.         self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep")
  44.         self.train_phase = tf.placeholder(tf.bool, name="train_phase")
  46.         self.weights = self._initialize_weights()
  48.         # model
  49.         self.embeddings = tf.nn.embedding_lookup(self.weights["feature_embeddings"],
  50.                                                          self.feat_index)  # None * F * K
  51.         feat_value = tf.reshape(self.feat_value, shape=[-1, self.field_size, 1])
  52.         self.embeddings = tf.multiply(self.embeddings, feat_value)
  54.         # ---------- first order term ----------
  55.         self.y_first_order = tf.nn.embedding_lookup(self.weights["feature_bias"], self.feat_index) # None * F * 1
  56.         self.y_first_order = tf.reduce_sum(tf.multiply(self.y_first_order, feat_value), 2)  # None * F
  57.         self.y_first_order = tf.nn.dropout(self.y_first_order, self.dropout_keep_fm[0]) # None * F
  59.         # ---------- second order term ---------------
  60.         # sum_square part
  61.         self.summed_features_emb = tf.reduce_sum(self.embeddings, 1)  # None * K
  62.         self.summed_features_emb_square = tf.square(self.summed_features_emb)  # None * K
  64.         # square_sum part
  65.         self.squared_features_emb = tf.square(self.embeddings)
  66.         self.squared_sum_features_emb = tf.reduce_sum(self.squared_features_emb, 1)  # None * K
  68.         # second order
  69.         self.y_second_order = 0.5 * tf.subtract(self.summed_features_emb_square, self.squared_sum_features_emb)  # None * K
  70.         self.y_second_order = tf.nn.dropout(self.y_second_order, self.dropout_keep_fm[1])  # None * K
  72.         # ---------- Deep component ----------
  73.         self.y_deep = tf.reshape(self.embeddings, shape=[-1, self.field_size * self.embedding_size]) # None * (F*K)
  74.         self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[0])
  75.         for i in range(0, len(self.deep_layers)):
  76.             self.y_deep = tf.add(tf.matmul(self.y_deep, self.weights["layer_%d" %i]), self.weights["bias_%d"%i]) # None * layer[i] * 1
  77.             if self.batch_norm:
  78.                 self.y_deep = self.batch_norm_layer(self.y_deep, train_phase=self.train_phase, scope_bn="bn_%d" %i) # None * layer[i] * 1
  79.             self.y_deep = self.deep_layers_activation(self.y_deep)
  80.             self.y_deep = tf.nn.dropout(self.y_deep, self.dropout_keep_deep[1+i]) # dropout at each Deep layer
  82.         # ---------- DeepFM ----------
  83.         if self.use_fm and self.use_deep:
  84.             concat_input = tf.concat([self.y_first_order, self.y_second_order, self.y_deep], axis=1)
  85.         elif self.use_fm:
  86.             concat_input = tf.concat([self.y_first_order, self.y_second_order], axis=1)
  87.         elif self.use_deep:
  88.             concat_input = self.y_deep
  89.         self.out = tf.add(tf.matmul(concat_input, self.weights["concat_projection"]), self.weights["concat_bias"])
  91.         # loss
  92.         if self.loss_type == "logloss":
  93.             self.out = tf.nn.sigmoid(self.out)
  94.             self.loss = tf.losses.log_loss(self.label, self.out)
  95.         elif self.loss_type == "mse":
  96.             self.loss = tf.nn.l2_loss(tf.subtract(self.label, self.out))
  97.         # l2 regularization on weights
  98.         if self.l2_reg > 0:
  99.             self.loss += tf.contrib.layers.l2_regularizer(
  100.                 self.l2_reg)(self.weights["concat_projection"])
  101.             if self.use_deep:
  102.                 for i in range(len(self.deep_layers)):
  103.                     self.loss += tf.contrib.layers.l2_regularizer(
  104.                         self.l2_reg)(self.weights["layer_%d"%i])
  106.         # optimizer
  107.         if self.optimizer_type == "adam":
  108.             self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, beta1=0.9, beta2=0.999,
  109.                                                     epsilon=1e-8).minimize(self.loss)
  110.         elif self.optimizer_type == "adagrad":
  111.             self.optimizer = tf.train.AdagradOptimizer(learning_rate=self.learning_rate,
  112.                                                        initial_accumulator_value=1e-8).minimize(self.loss)
  113.         elif self.optimizer_type == "gd":
  114.             self.optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate).minimize(self.loss)
  115.         elif self.optimizer_type == "momentum":
  116.             self.optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.95).minimize(
  117.                 self.loss)
  118.         elif self.optimizer_type == "yellowfin":
  119.             self.optimizer = YFOptimizer(learning_rate=self.learning_rate, momentum=0.0).minimize(
  120.                 self.loss)
  122.         # init
  123.         self.saver = tf.train.Saver()
  124.         init = tf.global_variables_initializer()
  125.         self.sess = self._init_session()
  126.         self.sess.run(init)
  128.         # number of params
  129.         total_parameters = 0
  130.         for variable in self.weights.values():
  131.             shape = variable.get_shape()
  132.             variable_parameters = 1
  133.             for dim in shape:
  134.                 variable_parameters *= dim.value
  135.             total_parameters += variable_parameters
  136.         if self.verbose > 0:
  137.             print("#params: %d" % total_parameters)
  140. def _init_session(self):
  141.     config = tf.ConfigProto(device_count={"gpu": 0})
  142.     config.gpu_options.allow_growth = True
  143.     return tf.Session(config=config)
  146. def _initialize_weights(self):
  147.     weights = dict()
  149.     # embeddings
  150.     weights["feature_embeddings"] = tf.Variable(
  151.         tf.random_normal([self.feature_size, self.embedding_size], 0.0, 0.01),
  152.         name="feature_embeddings")  # feature_size * K
  153.     weights["feature_bias"] = tf.Variable(
  154.         tf.random_uniform([self.feature_size, 1], 0.0, 1.0), name="feature_bias")  # feature_size * 1
  156.     # deep layers
  157.     num_layer = len(self.deep_layers)
  158.     input_size = self.field_size * self.embedding_size
  159.     glorot = np.sqrt(2.0 / (input_size + self.deep_layers[0]))
  160.     weights["layer_0"] = tf.Variable(
  161.         np.random.normal(loc=0, scale=glorot, size=(input_size, self.deep_layers[0])), dtype=np.float32)
  162.     weights["bias_0"] = tf.Variable(np.random.normal(loc=0, scale=glorot, size=(1, self.deep_layers[0])),
  163.                                                     dtype=np.float32)  # 1 * layers[0]
  164.     for i in range(1, num_layer):
  165.         glorot = np.sqrt(2.0 / (self.deep_layers[i-1] + self.deep_layers[i]))
  166.         weights["layer_%d" % i] = tf.Variable(
  167.             np.random.normal(loc=0, scale=glorot, size=(self.deep_layers[i-1], self.deep_layers[i])),
  168.             dtype=np.float32)  # layers[i-1] * layers[i]
  169.         weights["bias_%d" % i] = tf.Variable(
  170.             np.random.normal(loc=0, scale=glorot, size=(1, self.deep_layers[i])),
  171.             dtype=np.float32)  # 1 * layer[i]
  173.     # final concat projection layer
  174.     if self.use_fm and self.use_deep:
  175.         input_size = self.field_size + self.embedding_size + self.deep_layers[-1]
  176.     elif self.use_fm:
  177.         input_size = self.field_size + self.embedding_size
  178.     elif self.use_deep:
  179.         input_size = self.deep_layers[-1]
  180.     glorot = np.sqrt(2.0 / (input_size + 1))
  181.     weights["concat_projection"] = tf.Variable(
  182.                     np.random.normal(loc=0, scale=glorot, size=(input_size, 1)),
  183.                     dtype=np.float32)  # layers[i-1]*layers[i]
  184.     weights["concat_bias"] = tf.Variable(tf.constant(0.01), dtype=np.float32)
  186.     return weights
  189. def batch_norm_layer(self, x, train_phase, scope_bn):
  190.     bn_train = batch_norm(x, decay=self.batch_norm_decay, center=True, scale=True, updates_collections=None,
  191.                           is_training=True, reuse=None, trainable=True, scope=scope_bn)
  192.     bn_inference = batch_norm(x, decay=self.batch_norm_decay, center=True, scale=True, updates_collections=None,
  193.                               is_training=False, reuse=True, trainable=True, scope=scope_bn)
  194.     z = tf.cond(train_phase, lambda: bn_train, lambda: bn_inference)
  195.     return z
  198. def get_batch(self, Xi, Xv, y, batch_size, index):
  199.     start = index * batch_size
  200.     end = (index+1) * batch_size
  201.     end = end if end < len(y) else len(y)
  202.     return Xi[start:end], Xv[start:end], [[y_] for y_ in y[start:end]]
  205. # shuffle three lists simutaneously
  206. def shuffle_in_unison_scary(self, a, b, c):
  207.     rng_state = np.random.get_state()
  208.     np.random.shuffle(a)
  209.     np.random.set_state(rng_state)
  210.     np.random.shuffle(b)
  211.     np.random.set_state(rng_state)
  212.     np.random.shuffle(c)
  215. def fit_on_batch(self, Xi, Xv, y):
  216.     feed_dict = {self.feat_index: Xi,
  217.                  self.feat_value: Xv,
  218.                  self.label: y,
  219.                  self.dropout_keep_fm: self.dropout_fm,
  220.                  self.dropout_keep_deep: self.dropout_deep,
  221.                  self.train_phase: True}
  222.     loss, opt = self.sess.run((self.loss, self.optimizer), feed_dict=feed_dict)
  223.     return loss
  226. def fit(self, Xi_train, Xv_train, y_train,
  227.         Xi_valid=None, Xv_valid=None, y_valid=None,
  228.         early_stopping=False, refit=False):
  229.     """
  230.     :param Xi_train: [[ind1_1, ind1_2, ...], [ind2_1, ind2_2, ...], ..., [indi_1, indi_2, ..., indi_j, ...], ...]
  231.                      indi_j is the feature index of feature field j of sample i in the training set
  232.     :param Xv_train: [[val1_1, val1_2, ...], [val2_1, val2_2, ...], ..., [vali_1, vali_2, ..., vali_j, ...], ...]
  233.                      vali_j is the feature value of feature field j of sample i in the training set
  234.                      vali_j can be either binary (1/0, for binary/categorical features) or float (e.g., 10.24, for numerical features)
  235.     :param y_train: label of each sample in the training set
  236.     :param Xi_valid: list of list of feature indices of each sample in the validation set
  237.     :param Xv_valid: list of list of feature values of each sample in the validation set
  238.     :param y_valid: label of each sample in the validation set
  239.     :param early_stopping: perform early stopping or not
  240.     :param refit: refit the model on the train+valid dataset or not
  241.     :return: None
  242.     """
  243.     has_valid = Xv_valid is not None
  244.     for epoch in range(self.epoch):
  245.         t1 = time()
  246.         self.shuffle_in_unison_scary(Xi_train, Xv_train, y_train)
  247.         total_batch = int(len(y_train) / self.batch_size)
  248.         for i in range(total_batch):
  249.             Xi_batch, Xv_batch, y_batch = self.get_batch(Xi_train, Xv_train, y_train, self.batch_size, i)
  250.             self.fit_on_batch(Xi_batch, Xv_batch, y_batch)
  252.         # evaluate training and validation datasets
  253.         train_result = self.evaluate(Xi_train, Xv_train, y_train)
  254.         self.train_result.append(train_result)
  255.         if has_valid:
  256.             valid_result = self.evaluate(Xi_valid, Xv_valid, y_valid)
  257.             self.valid_result.append(valid_result)
  258.         if self.verbose > 0 and epoch % self.verbose == 0:
  259.             if has_valid:
  260.                 print("[%d] train-result=%.4f, valid-result=%.4f [%.1f s]"
  261.                     % (epoch + 1, train_result, valid_result, time() - t1))
  262.             else:
  263.                 print("[%d] train-result=%.4f [%.1f s]"
  264.                     % (epoch + 1, train_result, time() - t1))
  265.         if has_valid and early_stopping and self.training_termination(self.valid_result):
  266.             break
  268.     # fit a few more epoch on train+valid until result reaches the best_train_score
  269.     if has_valid and refit:
  270.         if self.greater_is_better:
  271.             best_valid_score = max(self.valid_result)
  272.         else:
  273.             best_valid_score = min(self.valid_result)
  274.         best_epoch = self.valid_result.index(best_valid_score)
  275.         best_train_score = self.train_result[best_epoch]
  276.         Xi_train = Xi_train + Xi_valid
  277.         Xv_train = Xv_train + Xv_valid
  278.         y_train = y_train + y_valid
  279.         for epoch in range(100):
  280.             self.shuffle_in_unison_scary(Xi_train, Xv_train, y_train)
  281.             total_batch = int(len(y_train) / self.batch_size)
  282.             for i in range(total_batch):
  283.                 Xi_batch, Xv_batch, y_batch = self.get_batch(Xi_train, Xv_train, y_train,
  284.                                                             self.batch_size, i)
  285.                 self.fit_on_batch(Xi_batch, Xv_batch, y_batch)
  286.             # check
  287.             train_result = self.evaluate(Xi_train, Xv_train, y_train)
  288.             if abs(train_result - best_train_score) < 0.001 or \
  289.                 (self.greater_is_better and train_result > best_train_score) or \
  290.                 ((not self.greater_is_better) and train_result < best_train_score):
  291.                 break
  294. def training_termination(self, valid_result):
  295.     if len(valid_result) > 5:
  296.         if self.greater_is_better:
  297.             if valid_result[-1] < valid_result[-2] and \
  298.                 valid_result[-2] < valid_result[-3] and \
  299.                 valid_result[-3] < valid_result[-4] and \
  300.                 valid_result[-4] < valid_result[-5]:
  301.                 return True
  302.         else:
  303.             if valid_result[-1] > valid_result[-2] and \
  304.                 valid_result[-2] > valid_result[-3] and \
  305.                 valid_result[-3] > valid_result[-4] and \
  306.                 valid_result[-4] > valid_result[-5]:
  307.                 return True
  308.     return False
  311. def predict(self, Xi, Xv):
  312.     """
  313.     :param Xi: list of list of feature indices of each sample in the dataset
  314.     :param Xv: list of list of feature values of each sample in the dataset
  315.     :return: predicted probability of each sample
  316.     """
  317.     # dummy y
  318.     dummy_y = [1] * len(Xi)
  319.     batch_index = 0
  320.     Xi_batch, Xv_batch, y_batch = self.get_batch(Xi, Xv, dummy_y, self.batch_size, batch_index)
  321.     y_pred = None
  322.     while len(Xi_batch) > 0:
  323.         num_batch = len(y_batch)
  324.         feed_dict = {self.feat_index: Xi_batch,
  325.                      self.feat_value: Xv_batch,
  326.                      self.label: y_batch,
  327.                      self.dropout_keep_fm: [1.0] * len(self.dropout_fm),
  328.                      self.dropout_keep_deep: [1.0] * len(self.dropout_deep),
  329.                      self.train_phase: False}
  330.         batch_out = self.sess.run(self.out, feed_dict=feed_dict)
  332.         if batch_index == 0:
  333.             y_pred = np.reshape(batch_out, (num_batch,))
  334.         else:
  335.             y_pred = np.concatenate((y_pred, np.reshape(batch_out, (num_batch,))))
  337.         batch_index += 1
  338.         Xi_batch, Xv_batch, y_batch = self.get_batch(Xi, Xv, dummy_y, self.batch_size, batch_index)
  340.     return y_pred
  343. def evaluate(self, Xi, Xv, y):
  344.     """
  345.     :param Xi: list of list of feature indices of each sample in the dataset
  346.     :param Xv: list of list of feature values of each sample in the dataset
  347.     :param y: label of each sample in the dataset
  348.     :return: metric of the evaluation
  349.     """
  350.     y_pred = self.predict(Xi, Xv)
  351.     return self.eval_metric(y, y_pred)

```