该项目的evaluate文件夹下的一个脚本eval_metric.py定义了关于测试过程中的评价函数。这个脚本主要涉及两个类：MApMetric和VOC07MApMetric，后者是继承前者并重写了一些方法得到的，因此MApMetric类是核心。这两者都是用来计算object detection算法中的MAp（Mean avearage precision）。

    1. import mxnet as mx
    2. import numpy as np
    3. import os
    4. import matplotlib
    5. matplotlib.use('Agg')
    6. import matplotlib.pyplot as plt
    8. class MApMetric(mx.metric.EvalMetric):
    9.     """
    10.     Calculate mean AP for object detection task
    12.     Parameters:
    13.     ---------
    14.     ovp_thresh : float
    15.         overlap threshold for TP
    16.     use_difficult : boolean
    17.         use difficult ground-truths if applicable, otherwise just ignore
    18.     class_names : list of str
    19.         optional, if provided, will print out AP for each class
    20.     pred_idx : int
    21.         prediction index in network output list
    22.     roc_output_path
    23.         optional, if provided, will save a ROC graph for each class
    24.     tensorboard_path
    25.         optional, if provided, will save a ROC graph to tensorboard
    26.     """
    27.     # __init__中还是执行常规的重置操作：reset()和一些赋值操作。
    28.     def __init__(self, ovp_thresh=0.5, use_difficult=False, class_names=None,
    29.                  pred_idx=0, roc_output_path=None, tensorboard_path=None):
    30.         super(MApMetric, self).__init__('mAP')
    31.         if class_names is None:
    32.             self.num = None
    33.         else:
    34.             assert isinstance(class_names, (list, tuple))
    35.             for name in class_names:
    36.                 assert isinstance(name, str), "must provide names as str"
    37.             num = len(class_names)
    38.             self.name = class_names + ['mAP']
    39.             self.num = num + 1
    40.         self.reset()
    41.         self.ovp_thresh = ovp_thresh
    42.         self.use_difficult = use_difficult
    43.         self.class_names = class_names
    44.         self.pred_idx = int(pred_idx)
    45.         self.roc_output_path = roc_output_path
    46.         self.tensorboard_path = tensorboard_path
    48.     def save_roc_graph(self, recall=None, prec=None, classkey=1, path=None, ap=None):
    49.         if not os.path.exists(path):
    50.             os.mkdir(path)
    51.         plot_path = os.path.join(path, 'roc_'+self.class_names[classkey])
    52.         if os.path.exists(plot_path):
    53.             os.remove(plot_path)
    54.         fig = plt.figure()
    55.         plt.title(self.class_names[classkey])
    56.         plt.plot(recall, prec, 'b', label='AP = %0.2f' % ap)
    57.         plt.legend(loc='lower right')
    58.         plt.xlim([0, 1])
    59.         plt.ylim([0, 1])
    60.         plt.ylabel('Precision')
    61.         plt.xlabel('Recall')
    62.         plt.savefig(plot_path)
    63.         plt.close(fig)
    65.     def reset(self):
    66.         """Clear the internal statistics to initial state."""
    67.         if getattr(self, 'num', None) is None:
    68.             self.num_inst = 0
    69.             self.sum_metric = 0.0
    70.         else:
    71.             self.num_inst = [0] * self.num
    72.             self.sum_metric = [0.0] * self.num
    73.         self.records = dict()
    74.         self.counts = dict()
    77. # 当代码要读取MAp值时就会调用get方法，在get方法中通过调用_update方法计算self.records变量得到MAp值。
    78. # 因为self.sum_metric和self.num_inst在这里是list，所以通过循环读取的方式最后返回tuple。
    79.     def get(self):
    80.         """Get the current evaluation result.
    82.         Returns
    83.         -------
    84.         name : str
    85.            Name of the metric.
    86.         value : float
    87.            Value of the evaluation.
    88.         """
    89.         self._update()  # update metric at this time
    90.         if self.num is None:
    91.             if self.num_inst == 0:
    92.                 return (self.name, float('nan'))
    93.             else:
    94.                 return (self.name, self.sum_metric / self.num_inst)
    95.         else:
    96.             names = ['%s'%(self.name[i]) for i in range(self.num)]
    97.             values = [x / y if y != 0 else float('nan') \
    98.                 for x, y in zip(self.sum_metric, self.num_inst)]
    99.             return (names, values)
    102. # update方法是更新MAp值的方法，目的是更新self.records变量。然后当代码要读取MAp值时就会调用get方法，
    103. # 在get方法中通过再调用_update方法计算self.records变量得到MAp值。
    104.     def update(self, labels, preds):
    105.         """
    106.         Update internal records. This function now only update internal buffer,
    107.         sum_metric and num_inst are updated in _update() function instead when
    108.         get() is called to return results.
    110.         Params:
    111.         ----------
    112.         labels: mx.nd.array (n * 6) or (n * 5), difficult column is optional
    113.             2-d array of ground-truths, n objects(id-xmin-ymin-xmax-ymax-[difficult])
    114.         preds: mx.nd.array (m * 6)
    115.             2-d array of detections, m objects(id-score-xmin-ymin-xmax-ymax)
    116.         """
    117. # IOU计算函数，就是计算两个框的交集面积除以并集面积的结果
    118.         def iou(x, ys):
    119.             """
    120.             Calculate intersection-over-union overlap
    121.             Params:
    122.             ----------
    123.             x : numpy.array
    124.                 single box [xmin, ymin ,xmax, ymax]
    125.             ys : numpy.array
    126.                 multiple box [[xmin, ymin, xmax, ymax], [...], ]
    127.             Returns:
    128.             -----------
    129.             numpy.array
    130.                 [iou1, iou2, ...], size == ys.shape[0]
    131.             """
    132.             ixmin = np.maximum(ys[:, 0], x[0])
    133.             iymin = np.maximum(ys[:, 1], x[1])
    134.             ixmax = np.minimum(ys[:, 2], x[2])
    135.             iymax = np.minimum(ys[:, 3], x[3])
    136.             iw = np.maximum(ixmax - ixmin, 0.)
    137.             ih = np.maximum(iymax - iymin, 0.)
    138.             inters = iw * ih
    139.             uni = (x[2] - x[0]) * (x[3] - x[1]) + (ys[:, 2] - ys[:, 0]) * \
    140.                 (ys[:, 3] - ys[:, 1]) - inters
    141.             ious = inters / uni
    142.             ious[uni < 1e-12] = 0  # in case bad boxes
    143.             return ious
    145.         # independant execution for each image
    146. # labels变量放的是batch size个图像的N个object的类别和坐标信息（非object的类别用-1表示），
    147. # preds则是网络的输出（包含4个，这里取最后一个得到batch size个图像的M个anchor的预测类别、置信度和坐标信息）。
    148. # 这个大的for循环就是循环batch中的每张图像。
    149.         for i in range(labels[0].shape[0]):
    150.             # get as numpy arrays
    151.             label = labels[0][i].asnumpy()
    152.             pred = preds[self.pred_idx][i].asnumpy()
    153.             # calculate for each class
    154.             while (pred.shape[0] > 0):
    155. # 每次循环都去pred（二维）的第一行的第一列，该值是第一个anchor的预测类别，后面会把属于该类别的预测值都copy到别的变量，
    156. # 然后将pred中该类别的预测值都删掉，所以每次循环时pred[0,0]的值都会变化，变化的次数就是你的类别数
    157.                 cid = int(pred[0, 0])
    158.                 indices = np.where(pred[:, 0].astype(int) == cid)[0]
    159. # 如果是背景类别，则从pred变量中删除
    160.                 if cid < 0:
    161.                     pred = np.delete(pred, indices, axis=0)
    162.                     continue
    163. # 将属于该预测类别的预测值copy给dets，然后从pred中删除该预测类别的预测值
    164.                 dets = pred[indices]
    165.                 pred = np.delete(pred, indices, axis=0)
    166.                 # sort by score, desceding
    167. # 按照置信度从高到低进行排序，records的第二列用来记录每个预测值的tp（truth positive）和fp（false positive）值，
    168. # 分别用1和2表示，初始化为0。
    169.                 dets[dets[:,1].argsort()[::-1]]
    170.                 records = np.hstack((dets[:, 1][:, np.newaxis], np.zeros((dets.shape[0], 1))))
    171.                 # ground-truths
    172. # label_indices是输入的该图像中object类别等于前面预测的cid类别的object index，并将这些object的类别和位置信息保存在gts变量中
    173.                 label_indices = np.where(label[:, 0].astype(int) == cid)[0]
    174.                 gts = label[label_indices, :]
    175.                 label = np.delete(label, label_indices, axis=0)
    176. # 如果真实的object类别和预测的cid类别有交集，则gts.size>0，否则跳过这个条件语句。
    177.                 if gts.size > 0:
    178.                     found = [False] * gts.shape[0]
    179. # 这个循环条件是遍历预测的类别值为cid的anchor，对每个anchor都计算其和真实的类别为cid的object框的IOU值。
    180. # 取其中最大的IOU值赋给ovmax
    181.                     for j in range(dets.shape[0]):
    182.                         # compute overlaps
    183.                         ious = iou(dets[j, 2:], gts[:, 1:5])
    184.                         ovargmax = np.argmax(ious)
    185.                         ovmax = ious[ovargmax]
    186. # 当IOU大于ovp_thresh时候，因为gts.shape[1]==5，所以执行 records[j, -1] = 1
    187. # 和found[ovargmax] = True。如果IOU没有达到这个阈值，则还是false positive。
    188.                         if ovmax > self.ovp_thresh:
    189.                             if (not self.use_difficult and
    190.                                 gts.shape[1] >= 6 and
    191.                                 gts[ovargmax, 5] > 0):
    192.                                 pass
    193.                             else:
    194.                                 if not found[ovargmax]:
    195.                                     records[j, -1] = 1  # tp
    196.                                     found[ovargmax] = True
    197.                                 else:
    198.                                     # duplicate
    199.                                     records[j, -1] = 2  # fp
    200.                         else:
    201. # 这里相当于预测的类别在图像的所有object类别中都不存在，所以都是false positive
    202.                             records[j, -1] = 2 # fp
    203.                 else:
    204.                     # no gt, mark all fp
    205.                     records[:, -1] = 2
    207.                 # ground truth count
    208.                 if (not self.use_difficult and gts.shape[1] >= 6):
    209.                     gt_count = np.sum(gts[:, 5] < 1)
    210.                 else:
    211.                     gt_count = gts.shape[0]
    213.                 # now we push records to buffer
    214.                 # first column: score, second column: tp/fp
    215.                 # 0: not set(matched to difficult or something), 1: tp, 2: fp
    216. # 过滤掉records中既不是fp也不是tp的预测值，然后将符合条件的records通过_insert方法插入到self.records，
    217. # 最后得到的self.records就是整个batch的总结果。
    218.                 records = records[np.where(records[:, -1] > 0)[0], :]
    219.                 if records.size > 0:
    220.                     self._insert(cid, records, gt_count)
    222.             # add missing class if not present in prediction
    223.             while (label.shape[0] > 0):
    224.                 cid = int(label[0, 0])
    225.                 label_indices = np.where(label[:, 0].astype(int) == cid)[0]
    226.                 label = np.delete(label, label_indices, axis=0)
    227.                 if cid < 0:
    228.                     continue
    229.                 gt_count = label_indices.size
    230.                 self._insert(cid, np.array([[0, 0]]), gt_count)
    233. #_update方法是作者自定义的一个内部方法，用来帮助算法在调用get方法的时候获取所需的计算值，要注意和update方法的差别。
    234. # 该方法基于前面update方法计算得到的sel.records来计算ap，self.records是一个包含number class个键值对的字典。
    235. # recall, prec = self._recall_prec(v, self.counts[k])是计算recall和precision，
    236. # ap = self._average_precision(recall, prec)是计算平均的recall和precision。
    237.     def _update(self):
    238.         """ update num_inst and sum_metric """
    239.         aps = []
    240.         for k, v in self.records.items():
    241.             recall, prec = self._recall_prec(v, self.counts[k])
    242.             ap = self._average_precision(recall, prec)
    243.             if self.roc_output_path is not None:
    244.                 self.save_roc_graph(recall=recall, prec=prec, classkey=k, path=self.roc_output_path, ap=ap)
    245.             aps.append(ap)
    246. # 因为k值是遍历所有object的类别，所以这里self.sum_metric[k]放的就是k这个类别的ap值。
    247. # 因此最后在界面上会显示每个类别的MAp值。
    248.             if self.num is not None and k < (self.num - 1):
    249.                 self.sum_metric[k] = ap
    250.                 self.num_inst[k] = 1
    251.         if self.num is None:
    252.             self.num_inst = 1
    253.             self.sum_metric = np.mean(aps)
    254. # 在sum_metric和self.num_inst的最后位置插入平均结果，所以在界面上会显示所有类别的平均MAp值。
    255.         else:
    256.             self.num_inst[-1] = 1
    257.             self.sum_metric[-1] = np.mean(aps)
    259. #_recall_prec方法是前面_update方法调用的一个辅助方法。
    260.     def _recall_prec(self, record, count):
    261.         """ get recall and precision from internal records """
    262.         record = np.delete(record, np.where(record[:, 1].astype(int) == 0)[0], axis=0)
    263.         sorted_records = record[record[:,0].argsort()[::-1]]
    264.         tp = np.cumsum(sorted_records[:, 1].astype(int) == 1)
    265.         fp = np.cumsum(sorted_records[:, 1].astype(int) == 2)
    266.         if count <= 0:
    267.             recall = tp * 0.0
    268.         else:
    269.             recall = tp / float(count)
    270.         prec = tp.astype(float) / (tp + fp)
    271.         return recall, prec
    273.     def _average_precision(self, rec, prec):
    274.         """
    275.         calculate average precision
    277.         Params:
    278.         ----------
    279.         rec : numpy.array
    280.             cumulated recall
    281.         prec : numpy.array
    282.             cumulated precision
    283.         Returns:
    284.         ----------
    285.         ap as float
    286.         """
    287.         # append sentinel values at both ends
    288.         mrec = np.concatenate(([0.], rec, [1.]))
    289.         mpre = np.concatenate(([0.], prec, [0.]))
    291.         # compute precision integration ladder
    292.         for i in range(mpre.size - 1, 0, -1):
    293.             mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
    295.         # look for recall value changes
    296.         i = np.where(mrec[1:] != mrec[:-1])[0]
    298.         # sum (\delta recall) * prec
    299.         ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
    300.         return ap
    302. # 将有效的records插入到self.records变量中
    303.     def _insert(self, key, records, count):
    304.         """ Insert records according to key """
    305.         if key not in self.records:
    306.             assert key not in self.counts
    307.             self.records[key] = records
    308.             self.counts[key] = count
    309.         else:
    310.             self.records[key] = np.vstack((self.records[key], records))
    311.             assert key in self.counts
    312.             self.counts[key] += count
    315. class VOC07MApMetric(MApMetric):
    316.     """ Mean average precision metric for PASCAL V0C 07 dataset """
    317.     def __init__(self, *args, **kwargs):
    318.         super(VOC07MApMetric, self).__init__(*args, **kwargs)
    320.     def _average_precision(self, rec, prec):
    321.         """
    322.         calculate average precision, override the default one,
    323.         special 11-point metric
    325.         Params:
    326.         ----------
    327.         rec : numpy.array
    328.             cumulated recall
    329.         prec : numpy.array
    330.             cumulated precision
    331.         Returns:
    332.         ----------
    333.         ap as float
    334.         """
    335.         ap = 0.
    336.         for t in np.arange(0., 1.1, 0.1):
    337.             if np.sum(rec >= t) == 0:
    338.                 p = 0
    339.             else:
    340.                 p = np.max(prec[rec >= t])
    341.             ap += p / 11.
    342.         return ap