From dcbc5b8256565433e09ae027f08acb15e4516027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20S=2E=20A?= Date: Sun, 2 Feb 2020 17:38:01 +0300 Subject: [PATCH 1/5] Adding support for F1 Score. --- keras_retinanet/bin/evaluate.py | 14 +++++++++++++- keras_retinanet/callbacks/eval.py | 30 +++++++++++++++++++++++++----- keras_retinanet/utils/eval.py | 29 +++++++++++++++++++++++++++-- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/keras_retinanet/bin/evaluate.py b/keras_retinanet/bin/evaluate.py index 1a10f2375..cfa360504 100755 --- a/keras_retinanet/bin/evaluate.py +++ b/keras_retinanet/bin/evaluate.py @@ -152,7 +152,7 @@ def main(args=None): from ..utils.coco_eval import evaluate_coco evaluate_coco(generator, model, args.score_threshold) else: - average_precisions, inference_time = evaluate( + average_precisions, inference_time, f1_scores = evaluate( generator, model, iou_threshold=args.iou_threshold, @@ -164,11 +164,19 @@ def main(args=None): # print evaluation total_instances = [] precisions = [] + scores = [] + for label, (average_precision, num_annotations) in average_precisions.items(): print('{:.0f} instances of class'.format(num_annotations), generator.label_to_name(label), 'with average precision: {:.4f}'.format(average_precision)) total_instances.append(num_annotations) precisions.append(average_precision) + + for label, (f1_score, num_annotations) in f1_scores.items(): + print('{:.0f} instances of class'.format(num_annotations), + generator.label_to_name(label), 'with F1 score: {:.4f}'.format(f1_score)) + # total_instances.append(num_annotations) + scores.append(f1_score) if sum(total_instances) == 0: print('No test instances found.') @@ -178,6 +186,10 @@ def main(args=None): print('mAP using the weighted average of precisions among classes: {:.4f}'.format(sum([a * b for a, b in zip(total_instances, precisions)]) / sum(total_instances))) print('mAP: {:.4f}'.format(sum(precisions) / sum(x > 0 for x in total_instances))) + + print('mF1 using the weighted F1 scores among classes: {:.4f}'.format(sum([a * b for a, b in zip(total_instances, scores)]) / sum(total_instances))) + print('mF1: {:.4f}'.format(sum(scores) / sum(x > 0 for x in total_instances))) + if __name__ == '__main__': diff --git a/keras_retinanet/callbacks/eval.py b/keras_retinanet/callbacks/eval.py index abdc8bbc0..d442bc13b 100644 --- a/keras_retinanet/callbacks/eval.py +++ b/keras_retinanet/callbacks/eval.py @@ -60,7 +60,7 @@ def on_epoch_end(self, epoch, logs=None): logs = logs or {} # run evaluation - average_precisions, _ = evaluate( + average_precisions, _, f1_scores = evaluate( self.generator, self.model, iou_threshold=self.iou_threshold, @@ -72,6 +72,8 @@ def on_epoch_end(self, epoch, logs=None): # compute per class average precision total_instances = [] precisions = [] + scores = [] + for label, (average_precision, num_annotations) in average_precisions.items(): if self.verbose == 1: print('{:.0f} instances of class'.format(num_annotations), @@ -83,16 +85,34 @@ def on_epoch_end(self, epoch, logs=None): else: self.mean_ap = sum(precisions) / sum(x > 0 for x in total_instances) + # compute per class F1 score + for label, (f1_score, num_annotations) in f1_scores.items(): + if self.verbose == 1: + print('{:.0f} instances of class'.format(num_annotations), + self.generator.label_to_name(label), ' with F1 score: {:.4f}'.format(f1_score)) + total_instances.append(num_annotations) + scores.append(f1_score) + + if self.weighted_average: + self.mean_f1_score = sum([a * b for a, b in zip(total_instances, scores)]) / sum(total_instances) + else: + self.mean_f1_score = sum(scores) / sum(x > 0 for x in total_instances) + if self.tensorboard: import tensorflow as tf if tf.version.VERSION < '2.0.0' and self.tensorboard.writer: summary = tf.Summary() - summary_value = summary.value.add() - summary_value.simple_value = self.mean_ap - summary_value.tag = "mAP" + summary_value_map = summary.value.add() + summary_value_map.simple_value = self.mean_ap + summary_value_map.tag = "mAP" + summary_value_f1 = summary.value.add() + summary_value_f1.simple_value = self.mean_f1_score + summary_value_f1.tag = "mF1" self.tensorboard.writer.add_summary(summary, epoch) logs['mAP'] = self.mean_ap + logs['mF1'] = self.mean_f1_score if self.verbose == 1: - print('mAP: {:.4f}'.format(self.mean_ap)) + print('mAP: {:.4f}\nmF1: {:.4f}'.format( + self.mean_ap, self.mean_f1_score)) diff --git a/keras_retinanet/utils/eval.py b/keras_retinanet/utils/eval.py index 6b709c59d..87b553fb2 100644 --- a/keras_retinanet/utils/eval.py +++ b/keras_retinanet/utils/eval.py @@ -56,6 +56,26 @@ def _compute_ap(recall, precision): return ap +def _compute_f1(recall, precision): + """ + # Arguments + recall: The recall curve (list). + precision: The precision curve (list). + # Returns + The F1 score + """ + + # to calculate area under PR curve, look for points + # where X axis (recall) and Y axis(precision) have + # higher and identical values + i = np.where(precision==precision.max() && recall==recall.max() && recall==precision)[0] + + # and multiply precision and recall, sum precision + # and recall, divide each other and multiply for two + f1 = np.sum(2 * (precision[i] * recall[i]) / (precision[i] + recall[i]), axis=0) + return f1.max() + + def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None): """ Get the detections from the model using the generator. @@ -174,6 +194,7 @@ def evaluate( all_detections, all_inferences = _get_detections(generator, model, score_threshold=score_threshold, max_detections=max_detections, save_path=save_path) all_annotations = _get_annotations(generator) average_precisions = {} + f1_scores = {} # all_detections = pickle.load(open('all_detections.pkl', 'rb')) # all_annotations = pickle.load(open('all_annotations.pkl', 'rb')) @@ -237,8 +258,12 @@ def evaluate( # compute average precision average_precision = _compute_ap(recall, precision) average_precisions[label] = average_precision, num_annotations - + # inference time inference_time = np.sum(all_inferences) / generator.size() + + # compute F1 scores + f1_score = _compute_f1(recall, precision) + f1_scores[label] = f1_score, num_annotations - return average_precisions, inference_time + return average_precisions, inference_time, f1_scores From 0201dab08ebb58ac551b50af70f13e935f8ff0db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20S=2E=20A?= Date: Sun, 2 Feb 2020 17:44:56 +0300 Subject: [PATCH 2/5] Using and operator. --- keras_retinanet/utils/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_retinanet/utils/eval.py b/keras_retinanet/utils/eval.py index 87b553fb2..8a2d1b0db 100644 --- a/keras_retinanet/utils/eval.py +++ b/keras_retinanet/utils/eval.py @@ -68,7 +68,7 @@ def _compute_f1(recall, precision): # to calculate area under PR curve, look for points # where X axis (recall) and Y axis(precision) have # higher and identical values - i = np.where(precision==precision.max() && recall==recall.max() && recall==precision)[0] + i = np.where(precision==precision.max() and recall==recall.max() and recall==precision)[0] # and multiply precision and recall, sum precision # and recall, divide each other and multiply for two From c95e1d1a0f1cd4afa84ff3882e061c58cb017173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20S=2E=20A?= Date: Sun, 2 Feb 2020 18:23:06 +0300 Subject: [PATCH 3/5] Fix and operator. --- keras_retinanet/utils/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_retinanet/utils/eval.py b/keras_retinanet/utils/eval.py index 8a2d1b0db..c89d11930 100644 --- a/keras_retinanet/utils/eval.py +++ b/keras_retinanet/utils/eval.py @@ -68,7 +68,7 @@ def _compute_f1(recall, precision): # to calculate area under PR curve, look for points # where X axis (recall) and Y axis(precision) have # higher and identical values - i = np.where(precision==precision.max() and recall==recall.max() and recall==precision)[0] + i = np.where(precision==precision.max() & recall==recall.max() & recall==precision)[0] # and multiply precision and recall, sum precision # and recall, divide each other and multiply for two From 6aeaebecd25f85117a6317d523d8873af9a3d3e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20S=2E=20A?= Date: Sun, 2 Feb 2020 18:28:18 +0300 Subject: [PATCH 4/5] Added missing parenthesis. --- keras_retinanet/utils/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_retinanet/utils/eval.py b/keras_retinanet/utils/eval.py index c89d11930..f3a26ba2d 100644 --- a/keras_retinanet/utils/eval.py +++ b/keras_retinanet/utils/eval.py @@ -68,7 +68,7 @@ def _compute_f1(recall, precision): # to calculate area under PR curve, look for points # where X axis (recall) and Y axis(precision) have # higher and identical values - i = np.where(precision==precision.max() & recall==recall.max() & recall==precision)[0] + i = np.where((precision==precision.max()) & (recall==recall.max()) & (recall==precision))[0] # and multiply precision and recall, sum precision # and recall, divide each other and multiply for two From 31a630dbc72fcbf850e19ceb724b96ad24ff4822 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20L=2E=20S=2E=20A?= Date: Mon, 10 Feb 2020 15:11:59 +0300 Subject: [PATCH 5/5] Simplest formula F1 works with this. --- keras_retinanet/utils/eval.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/keras_retinanet/utils/eval.py b/keras_retinanet/utils/eval.py index f3a26ba2d..278ee5f4b 100644 --- a/keras_retinanet/utils/eval.py +++ b/keras_retinanet/utils/eval.py @@ -68,12 +68,14 @@ def _compute_f1(recall, precision): # to calculate area under PR curve, look for points # where X axis (recall) and Y axis(precision) have # higher and identical values - i = np.where((precision==precision.max()) & (recall==recall.max()) & (recall==precision))[0] + # i = np.where((precision==precision.max()) & (recall==recall.max()) & (recall==precision))[0] # and multiply precision and recall, sum precision # and recall, divide each other and multiply for two - f1 = np.sum(2 * (precision[i] * recall[i]) / (precision[i] + recall[i]), axis=0) - return f1.max() + # f1 = np.sum(2 * (precision[i] * recall[i]) / (precision[i] + recall[i]), axis=0) + + f1 = 2*((precision.max()*recall.max())/(precision.max()+recall.max())) + return f1 # f1.max() def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None):