diff --git a/docs/source/apps.rst b/docs/source/apps.rst index 255db787c2..3bb85c9296 100644 --- a/docs/source/apps.rst +++ b/docs/source/apps.rst @@ -177,3 +177,10 @@ Applications ~~~~~~~~~~~~~~~~~~~~~~~~ .. automodule:: monai.apps.detection.utils.box_selector :members: + +`Detection metrics` +~~~~~~~~~~~~~~~~~~~ +.. automodule:: monai.apps.detection.metrics.coco + :members: +.. automodule:: monai.apps.detection.metrics.matching + :members: diff --git a/monai/apps/detection/metrics/__init__.py b/monai/apps/detection/metrics/__init__.py new file mode 100644 index 0000000000..1e97f89407 --- /dev/null +++ b/monai/apps/detection/metrics/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/monai/apps/detection/metrics/coco.py b/monai/apps/detection/metrics/coco.py new file mode 100644 index 0000000000..7c6b02fe4f --- /dev/null +++ b/monai/apps/detection/metrics/coco.py @@ -0,0 +1,547 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ========================================================================= +# Adapted from https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/coco.py +# which has the following license... +# https://github.com/MIC-DKFZ/nnDetection/blob/main/LICENSE +# +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ========================================================================= +# Adapted from https://github.com/cocodataset/cocoapi +# which has the following license... +# https://github.com/cocodataset/cocoapi/blob/master/license.txt + +# Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +""" +This script is almost same with https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/coco.py +The changes include 1) code reformatting, 2) docstrings. +""" + +import logging as logger +import time +from typing import Dict, List, Sequence, Tuple, Union + +import numpy as np + + +class COCOMetric: + def __init__( + self, + classes: Sequence[str], + iou_list: Sequence[float] = (0.1, 0.5, 0.75), + iou_range: Sequence[float] = (0.1, 0.5, 0.05), + max_detection: Sequence[int] = (1, 5, 100), + per_class: bool = True, + verbose: bool = True, + ): + """ + Class to compute COCO metrics + Metrics computed includes, + + - mAP over the IoU range specified by `iou_range` at last value of `max_detection` + - AP values at IoU thresholds specified by `iou_list` at last value of `max_detection` + - AR over max detections thresholds defined by `max_detection` (over iou range) + + Args: + classes (Sequence[str]): name of each class (index needs to correspond to predicted class indices!) + iou_list (Sequence[float]): specific thresholds where ap is evaluated and saved + iou_range (Sequence[float]): (start, stop, step) for mAP iou thresholds + max_detection (Sequence[int]): maximum number of detections per image + verbose (bool): log time needed for evaluation + + Example: + + .. code-block:: python + + from monai.data.box_utils import box_iou + from monai.apps.detection.metrics.coco import COCOMetric + from monai.apps.detection.metrics.matching import matching_batch + # 3D example outputs of one image from detector + val_outputs_all = [ + {"boxes": torch.tensor([[1,1,1,3,4,5]],dtype=torch.float16), + "labels": torch.randint(3,(1,)), + "scores": torch.randn((1,)).absolute()}, + ] + val_targets_all = [ + {"boxes": torch.tensor([[1,1,1,2,6,4]],dtype=torch.float16), + "labels": torch.randint(3,(1,))}, + ] + + coco_metric = COCOMetric( + classes=['c0','c1','c2'], iou_list=[0.1], max_detection=[10] + ) + results_metric = matching_batch( + iou_fn=box_iou, + iou_thresholds=coco_metric.iou_thresholds, + pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all], + pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all], + pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all], + gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all], + gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all], + ) + val_metric_dict = coco_metric(results_metric) + print(val_metric_dict) + """ + self.verbose = verbose + self.classes = classes + self.per_class = per_class + + iou_list_np = np.array(iou_list) + _iou_range = np.linspace( + iou_range[0], iou_range[1], int(np.round((iou_range[1] - iou_range[0]) / iou_range[2])) + 1, endpoint=True + ) + self.iou_thresholds = np.union1d(iou_list_np, _iou_range) + self.iou_range = iou_range + + # get indices of iou values of ious range and ious list for later evaluation + self.iou_list_idx = np.nonzero(iou_list_np[:, np.newaxis] == self.iou_thresholds[np.newaxis])[1] + self.iou_range_idx = np.nonzero(_iou_range[:, np.newaxis] == self.iou_thresholds[np.newaxis])[1] + + if ( + not (self.iou_thresholds[self.iou_list_idx] == iou_list_np).all() + or not (self.iou_thresholds[self.iou_range_idx] == _iou_range).all() + ): + raise ValueError( + "Require self.iou_thresholds[self.iou_list_idx] == iou_list_np and " + "self.iou_thresholds[self.iou_range_idx] == _iou_range." + ) + + self.recall_thresholds = np.linspace(0.0, 1.00, int(np.round((1.00 - 0.0) / 0.01)) + 1, endpoint=True) + self.max_detections = max_detection + + def __call__(self, *args, **kwargs) -> Tuple[Dict[str, float], Union[Dict[str, np.ndarray], None]]: + """ + Compute metric. See :func:`compute` for more information. + + Args: + *args: positional arguments passed to :func:`compute` + **kwargs: keyword arguments passed to :func:`compute` + + Returns: + Dict[str, float]: dictionary with scalar values for evaluation + Dict[str, np.ndarray]: dictionary with arrays, e.g. for visualization of graphs + """ + return self.compute(*args, **kwargs) + + def check_number_of_iou(self, *args) -> None: + """ + Check if shape of input in first dimension is consistent with expected IoU values + (assumes IoU dimension is the first dimension) + + Args: + args: array like inputs with shape function + """ + num_ious = len(self.get_iou_thresholds()) + for arg in args: + if arg.shape[0] != num_ious: + raise ValueError( + f"Require arg.shape[0] == len(self.get_iou_thresholds()). Got arg.shape[0]={arg.shape[0]}, " + f"self.get_iou_thresholds()={self.get_iou_thresholds()}." + ) + + def get_iou_thresholds(self) -> Sequence[float]: + """ + Return IoU thresholds needed for this metric in an numpy array + + Returns: + Sequence[float]: IoU thresholds [M], M is the number of thresholds + """ + return list(self.iou_thresholds) + + def compute(self, results_list: List[Dict[int, Dict[str, np.ndarray]]]) -> Tuple[Dict[str, float], None]: + """ + Compute COCO metrics + + Args: + results_list (List[Dict[int, Dict[str, np.ndarray]]]): list with results per image (in list) + per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`. + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + + Returns: + Dict[str, float], dictionary with coco metrics + """ + if self.verbose: + logger.info("Start COCO metric computation...") + tic = time.time() + + dataset_statistics = self._compute_statistics(results_list=results_list) # Dict[str, Union[np.ndarray, List]] + + if self.verbose: + toc = time.time() + logger.info(f"Statistics for COCO metrics finished (t={(toc - tic):0.2f}s).") + + results = {} + results.update(self._compute_ap(dataset_statistics)) + results.update(self._compute_ar(dataset_statistics)) + + if self.verbose: + toc = time.time() + logger.info(f"COCO metrics computed in t={(toc - tic):0.2f}s.") + return results, None + + def _compute_ap(self, dataset_statistics: Dict[str, Union[np.ndarray, List]]) -> Dict[str, float]: + """ + Compute AP metrics + + Args: + dataset_statistics (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list) + per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`. + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + """ + results = {} + if self.iou_range: # mAP + key = ( + f"mAP_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_" + f"MaxDet_{self.max_detections[-1]}" + ) + results[key] = self._select_ap(dataset_statistics, iou_idx=self.iou_range_idx, max_det_idx=-1) + + if self.per_class: + for cls_idx, cls_str in enumerate(self.classes): # per class results + key = ( + f"{cls_str}_" + f"mAP_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_" + f"MaxDet_{self.max_detections[-1]}" + ) + results[key] = self._select_ap( + dataset_statistics, iou_idx=self.iou_range_idx, cls_idx=cls_idx, max_det_idx=-1 + ) + + for idx in self.iou_list_idx: # AP@IoU + key = f"AP_IoU_{self.iou_thresholds[idx]:.2f}_MaxDet_{self.max_detections[-1]}" + results[key] = self._select_ap(dataset_statistics, iou_idx=[idx], max_det_idx=-1) + + if self.per_class: + for cls_idx, cls_str in enumerate(self.classes): # per class results + key = f"{cls_str}_" f"AP_IoU_{self.iou_thresholds[idx]:.2f}_" f"MaxDet_{self.max_detections[-1]}" + results[key] = self._select_ap(dataset_statistics, iou_idx=[idx], cls_idx=cls_idx, max_det_idx=-1) + return results + + def _compute_ar(self, dataset_statistics: Dict[str, Union[np.ndarray, List]]) -> Dict[str, float]: + """ + Compute AR metrics + + Args: + dataset_statistics (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list) + per category (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`. + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + """ + results = {} + for max_det_idx, max_det in enumerate(self.max_detections): # mAR + key = f"mAR_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_MaxDet_{max_det}" + results[key] = self._select_ar(dataset_statistics, max_det_idx=max_det_idx) + + if self.per_class: + for cls_idx, cls_str in enumerate(self.classes): # per class results + key = ( + f"{cls_str}_" + f"mAR_IoU_{self.iou_range[0]:.2f}_{self.iou_range[1]:.2f}_{self.iou_range[2]:.2f}_" + f"MaxDet_{max_det}" + ) + results[key] = self._select_ar(dataset_statistics, cls_idx=cls_idx, max_det_idx=max_det_idx) + + for idx in self.iou_list_idx: # AR@IoU + key = f"AR_IoU_{self.iou_thresholds[idx]:.2f}_MaxDet_{self.max_detections[-1]}" + results[key] = self._select_ar(dataset_statistics, iou_idx=idx, max_det_idx=-1) + + if self.per_class: + for cls_idx, cls_str in enumerate(self.classes): # per class results + key = f"{cls_str}_" f"AR_IoU_{self.iou_thresholds[idx]:.2f}_" f"MaxDet_{self.max_detections[-1]}" + results[key] = self._select_ar(dataset_statistics, iou_idx=idx, cls_idx=cls_idx, max_det_idx=-1) + return results + + @staticmethod + def _select_ap( + dataset_statistics: dict, + iou_idx: Union[int, List[int], np.ndarray, None] = None, + cls_idx: Union[int, Sequence[int], None] = None, + max_det_idx: int = -1, + ) -> float: + """ + Compute average precision + + Args: + dataset_statistics (dict): computed statistics over dataset + + - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max + detection thresholds + - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections] + - `precision`: Precision values at specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + - `scores`: Scores corresponding to specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + iou_idx: index of IoU values to select for evaluation(if None, all values are used) + cls_idx: class indices to select, if None all classes will be selected + max_det_idx (int): index to select max detection threshold from data + + Returns: + np.ndarray: AP value + """ + prec = dataset_statistics["precision"] + if iou_idx is not None: + prec = prec[iou_idx] + if cls_idx is not None: + prec = prec[..., cls_idx, :] + prec = prec[..., max_det_idx] + return float(np.mean(prec)) + + @staticmethod + def _select_ar( + dataset_statistics: dict, + iou_idx: Union[int, Sequence[int], None] = None, + cls_idx: Union[int, Sequence[int], None] = None, + max_det_idx: int = -1, + ) -> float: + """ + Compute average recall + + Args: + dataset_statistics (dict): computed statistics over dataset + + - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max + detection thresholds + - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections] + - `precision`: Precision values at specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + - `scores`: Scores corresponding to specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + iou_idx: index of IoU values to select for evaluation(if None, all values are used) + cls_idx: class indices to select, if None all classes will be selected + max_det_idx (int): index to select max detection threshold from data + + Returns: + np.ndarray: recall value + """ + rec = dataset_statistics["recall"] + if iou_idx is not None: + rec = rec[iou_idx] + if cls_idx is not None: + rec = rec[..., cls_idx, :] + rec = rec[..., max_det_idx] + + if len(rec[rec > -1]) == 0: + return -1.0 + + return float(np.mean(rec[rec > -1])) + + def _compute_statistics( + self, results_list: List[Dict[int, Dict[str, np.ndarray]]] + ) -> Dict[str, Union[np.ndarray, List]]: + """ + Compute statistics needed for COCO metrics (mAP, AP of individual classes, mAP@IoU_Thresholds, AR) + Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py + + Args: + results_list (List[Dict[int, Dict[str, np.ndarray]]]): list with result s per image (in list) + per cateory (dict). Inner Dict contains multiple results obtained by :func:`box_matching_batch`. + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + + Returns: + dict: computed statistics over dataset + - `counts`: Number of thresholds, Number recall thresholds, Number of classes, Number of max + detection thresholds + - `recall`: Computed recall values [num_iou_th, num_classes, num_max_detections] + - `precision`: Precision values at specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + - `scores`: Scores corresponding to specified recall thresholds + [num_iou_th, num_recall_th, num_classes, num_max_detections] + """ + num_iou_th = len(self.iou_thresholds) + num_recall_th = len(self.recall_thresholds) + num_classes = len(self.classes) + num_max_detections = len(self.max_detections) + + # -1 for the precision of absent categories + precision = -np.ones((num_iou_th, num_recall_th, num_classes, num_max_detections)) + recall = -np.ones((num_iou_th, num_classes, num_max_detections)) + scores = -np.ones((num_iou_th, num_recall_th, num_classes, num_max_detections)) + + for cls_idx, cls_i in enumerate(self.classes): # for each class + for max_det_idx, max_det in enumerate(self.max_detections): # for each maximum number of detections + results = [r[cls_idx] for r in results_list if cls_idx in r] # len is num_images + + if len(results) == 0: + logger.warning(f"WARNING, no results found for coco metric for class {cls_i}") + continue + + dt_scores = np.concatenate([r["dtScores"][0:max_det] for r in results]) + # different sorting method generates slightly different results. + # mergesort is used to be consistent as Matlab implementation. + inds = np.argsort(-dt_scores, kind="mergesort") + dt_scores_sorted = dt_scores[inds] + + # r['dtMatches'] [T, R], where R = sum(all detections) + dt_matches = np.concatenate([r["dtMatches"][:, 0:max_det] for r in results], axis=1)[:, inds] + dt_ignores = np.concatenate([r["dtIgnore"][:, 0:max_det] for r in results], axis=1)[:, inds] + self.check_number_of_iou(dt_matches, dt_ignores) + gt_ignore = np.concatenate([r["gtIgnore"] for r in results]) + num_gt = np.count_nonzero(gt_ignore == 0) # number of ground truth boxes (non ignored) + if num_gt == 0: + logger.warning(f"WARNING, no gt found for coco metric for class {cls_i}") + continue + + # ignore cases need to be handled differently for tp and fp + tps = np.logical_and(dt_matches, np.logical_not(dt_ignores)) + fps = np.logical_and(np.logical_not(dt_matches), np.logical_not(dt_ignores)) + + tp_sum = np.cumsum(tps, axis=1).astype(dtype=np.float32) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=np.float32) + + for th_ind, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): # for each threshold th_ind + tp, fp = np.array(tp), np.array(fp) + r, p, s = _compute_stats_single_threshold(tp, fp, dt_scores_sorted, self.recall_thresholds, num_gt) + recall[th_ind, cls_idx, max_det_idx] = r + precision[th_ind, :, cls_idx, max_det_idx] = p + # corresponding score thresholds for recall steps + scores[th_ind, :, cls_idx, max_det_idx] = s + + return { + "counts": [num_iou_th, num_recall_th, num_classes, num_max_detections], # [4] + "recall": recall, # [num_iou_th, num_classes, num_max_detections] + "precision": precision, # [num_iou_th, num_recall_th, num_classes, num_max_detections] + "scores": scores, # [num_iou_th, num_recall_th, num_classes, num_max_detections] + } + + +def _compute_stats_single_threshold( + tp: np.ndarray, + fp: np.ndarray, + dt_scores_sorted: np.ndarray, + recall_thresholds: Union[np.ndarray, Sequence[float]], + num_gt: int, +) -> Tuple[float, np.ndarray, np.ndarray]: + """ + Compute recall value, precision curve and scores thresholds + Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py + + Args: + tp (np.ndarray): cumsum over true positives [R], R is the number of detections + fp (np.ndarray): cumsum over false positives [R], R is the number of detections + dt_scores_sorted (np.ndarray): sorted (descending) scores [R], R is the number of detections + recall_thresholds (Sequence[float]): recall thresholds which should be evaluated + num_gt (int): number of ground truth bounding boxes (excluding boxes which are ignored) + + Returns: + - float, overall recall for given IoU value + - np.ndarray, precision values at defined recall values + [RTH], where RTH is the number of recall thresholds + - np.ndarray, prediction scores corresponding to recall values + [RTH], where RTH is the number of recall thresholds + """ + num_recall_th = len(recall_thresholds) + + rc = tp / num_gt + # np.spacing(1) is the smallest representable epsilon with float + pr = tp / (fp + tp + np.spacing(1)) + + if len(tp): + recall = rc[-1] + else: + # no prediction + recall = 0 + + # array where precision values nearest to given recall th are saved + precision = np.zeros((num_recall_th,)) + # save scores for corresponding recall value in here + th_scores = np.zeros((num_recall_th,)) + # numpy is slow without cython optimization for accessing elements + # use python array gets significant speed improvement + pr = pr.tolist() + precision = precision.tolist() + + # smooth precision curve (create box shape) + for i in range(len(tp) - 1, 0, -1): + if pr[i] > pr[i - 1]: + pr[i - 1] = pr[i] + + # get indices to nearest given recall threshold (nn interpolation!) + inds = np.searchsorted(rc, recall_thresholds, side="left") + try: + for save_idx, array_index in enumerate(inds): + precision[save_idx] = pr[array_index] + th_scores[save_idx] = dt_scores_sorted[array_index] + except BaseException: + pass + + return recall, np.array(precision), np.array(th_scores) diff --git a/monai/apps/detection/metrics/matching.py b/monai/apps/detection/metrics/matching.py new file mode 100644 index 0000000000..6df026bf54 --- /dev/null +++ b/monai/apps/detection/metrics/matching.py @@ -0,0 +1,367 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ========================================================================= +# Adapted from https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/matching.py +# which has the following license... +# https://github.com/MIC-DKFZ/nnDetection/blob/main/LICENSE +# +# Copyright 2020 Division of Medical Image Computing, German Cancer Research Center (DKFZ), Heidelberg, Germany +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ========================================================================= +# Adapted from https://github.com/cocodataset/cocoapi +# which has the following license... +# https://github.com/cocodataset/cocoapi/blob/master/license.txt + +# Copyright (c) 2014, Piotr Dollar and Tsung-Yi Lin +# All rights reserved. + +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: + +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# The views and conclusions contained in the software and documentation are those +# of the authors and should not be interpreted as representing official policies, +# either expressed or implied, of the FreeBSD Project. + +""" +This script is almost same with https://github.com/MIC-DKFZ/nnDetection/blob/main/nndet/evaluator/detection/matching.py +The changes include 1) code reformatting, 2) docstrings, +3) allow input args gt_ignore to be optional. (If so, no GT boxes will be ignored.) +""" + +from typing import Callable, Dict, List, Sequence, Union + +import numpy as np + +__all__ = ["matching_batch"] + + +def matching_batch( + iou_fn: Callable[[np.ndarray, np.ndarray], np.ndarray], + iou_thresholds: Sequence[float], + pred_boxes: Sequence[np.ndarray], + pred_classes: Sequence[np.ndarray], + pred_scores: Sequence[np.ndarray], + gt_boxes: Sequence[np.ndarray], + gt_classes: Sequence[np.ndarray], + gt_ignore: Union[Sequence[Sequence[bool]], Sequence[np.ndarray], None] = None, + max_detections: int = 100, +) -> List[Dict[int, Dict[str, np.ndarray]]]: + """ + Match boxes of a batch to corresponding ground truth for each category + independently. + + Args: + iou_fn: compute overlap for each pair + iou_thresholds: defined which IoU thresholds should be evaluated + pred_boxes: predicted boxes from single batch; List[[D, dim * 2]], + D number of predictions + pred_classes: predicted classes from a single batch; List[[D]], + D number of predictions + pred_scores: predicted score for each bounding box; List[[D]], + D number of predictions + gt_boxes: ground truth boxes; List[[G, dim * 2]], G number of ground + truth + gt_classes: ground truth classes; List[[G]], G number of ground truth + gt_ignore: specified if which ground truth boxes are not counted as + true positives. If not given, when use all the gt_boxes. + (detections which match theses boxes are not counted as false + positives either); List[[G]], G number of ground truth + max_detections: maximum number of detections which should be evaluated + + Returns: + List[Dict[int, Dict[str, np.ndarray]]], each Dict[str, np.ndarray] corresponds to an image. + Dict has the following keys. + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + + Example: + + .. code-block:: python + + from monai.data.box_utils import box_iou + from monai.apps.detection.metrics.coco import COCOMetric + from monai.apps.detection.metrics.matching import matching_batch + # 3D example outputs of one image from detector + val_outputs_all = [ + {"boxes": torch.tensor([[1,1,1,3,4,5]],dtype=torch.float16), + "labels": torch.randint(3,(1,)), + "scores": torch.randn((1,)).absolute()}, + ] + val_targets_all = [ + {"boxes": torch.tensor([[1,1,1,2,6,4]],dtype=torch.float16), + "labels": torch.randint(3,(1,))}, + ] + + coco_metric = COCOMetric( + classes=['c0','c1','c2'], iou_list=[0.1], max_detection=[10] + ) + results_metric = matching_batch( + iou_fn=box_iou, + iou_thresholds=coco_metric.iou_thresholds, + pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all], + pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all], + pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all], + gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all], + gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all], + ) + val_metric_dict = coco_metric(results_metric) + print(val_metric_dict) + """ + results = [] + if gt_ignore is None: + gt_ignore = [np.full_like(gt_c, False) for gt_c in gt_classes] + # iterate over images/batches + for pboxes, pclasses, pscores, gboxes, gclasses, gignore in zip( + pred_boxes, pred_classes, pred_scores, gt_boxes, gt_classes, gt_ignore + ): + # for each image + img_classes = np.union1d(pclasses, gclasses) # possible class labels + result = {} # dict contains results for each class in one image + for c in img_classes: + pred_mask = pclasses == c # bool mask predictions with current class + gt_mask = gclasses == c # nool mask ground trtuh with current class + + if not np.any(gt_mask): # no ground truth + result[c] = _matching_no_gt( + iou_thresholds=iou_thresholds, pred_scores=pscores[pred_mask], max_detections=max_detections + ) + elif not np.any(pred_mask): # no predictions + result[c] = _matching_no_pred(iou_thresholds=iou_thresholds, gt_ignore=gignore[gt_mask]) + else: # at least one prediction and one ground truth + result[c] = _matching_single_image_single_class( + iou_fn=iou_fn, + pred_boxes=pboxes[pred_mask], + pred_scores=pscores[pred_mask], + gt_boxes=gboxes[gt_mask], + gt_ignore=gignore[gt_mask], + max_detections=max_detections, + iou_thresholds=iou_thresholds, + ) + results.append(result) + return results + + +def _matching_no_gt( + iou_thresholds: Sequence[float], pred_scores: np.ndarray, max_detections: int +) -> Dict[str, np.ndarray]: + """ + Matching result with not ground truth in image + + Args: + iou_thresholds: defined which IoU thresholds should be evaluated + dt_scores: predicted scores + max_detections: maximum number of allowed detections per image. + This functions uses this parameter to stay consistent with + the actual matching function which needs this limit. + + Returns: + computed matching, a Dict[str, np.ndarray] + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + """ + dt_ind = np.argsort(-pred_scores, kind="mergesort") + dt_ind = dt_ind[:max_detections] + dt_scores = pred_scores[dt_ind] + + num_preds = len(dt_scores) + + gt_match: np.ndarray = np.array([[]] * len(iou_thresholds)) + dt_match: np.ndarray = np.zeros((len(iou_thresholds), num_preds)) + dt_ignore: np.ndarray = np.zeros((len(iou_thresholds), num_preds)) + + return { + "dtMatches": dt_match, # [T, D], where T = number of thresholds, D = number of detections + "gtMatches": gt_match, # [T, G], where T = number of thresholds, G = number of ground truth + "dtScores": dt_scores, # [D] detection scores + "gtIgnore": np.array([]).reshape(-1), # [G] indicate whether ground truth should be ignored + "dtIgnore": dt_ignore, # [T, D], indicate which detections should be ignored + } + + +def _matching_no_pred(iou_thresholds: Sequence[float], gt_ignore: np.ndarray) -> Dict[str, np.ndarray]: + """ + Matching result with no predictions + + Args: + iou_thresholds: defined which IoU thresholds should be evaluated + gt_ignore: specified if which ground truth boxes are not counted as + true positives (detections which match theses boxes are not + counted as false positives either); [G], G number of ground truth + + Returns: + dict: computed matching + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + """ + dt_scores: np.ndarray = np.array([]) + dt_match: np.ndarray = np.array([[]] * len(iou_thresholds)) + dt_ignore: np.ndarray = np.array([[]] * len(iou_thresholds)) + + n_gt = 0 if gt_ignore.size == 0 else gt_ignore.shape[0] + gt_match = np.zeros((len(iou_thresholds), n_gt)) + + return { + "dtMatches": dt_match, # [T, D], where T = number of thresholds, D = number of detections + "gtMatches": gt_match, # [T, G], where T = number of thresholds, G = number of ground truth + "dtScores": dt_scores, # [D] detection scores + "gtIgnore": gt_ignore.reshape(-1), # [G] indicate whether ground truth should be ignored + "dtIgnore": dt_ignore, # [T, D], indicate which detections should be ignored + } + + +def _matching_single_image_single_class( + iou_fn: Callable[[np.ndarray, np.ndarray], np.ndarray], + pred_boxes: np.ndarray, + pred_scores: np.ndarray, + gt_boxes: np.ndarray, + gt_ignore: np.ndarray, + max_detections: int, + iou_thresholds: Sequence[float], +) -> Dict[str, np.ndarray]: + """ + Adapted from https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/cocoeval.py + + Args: + iou_fn: compute overlap for each pair + iou_thresholds: defined which IoU thresholds should be evaluated + pred_boxes: predicted boxes from single batch; [D, dim * 2], D number + of predictions + pred_scores: predicted score for each bounding box; [D], D number of + predictions + gt_boxes: ground truth boxes; [G, dim * 2], G number of ground truth + gt_ignore: specified if which ground truth boxes are not counted as + true positives (detections which match theses boxes are not + counted as false positives either); [G], G number of ground truth + max_detections: maximum number of detections which should be evaluated + + Returns: + dict: computed matching + + - `dtMatches`: matched detections [T, D], where T = number of + thresholds, D = number of detections + - `gtMatches`: matched ground truth boxes [T, G], where T = number + of thresholds, G = number of ground truth + - `dtScores`: prediction scores [D] detection scores + - `gtIgnore`: ground truth boxes which should be ignored + [G] indicate whether ground truth should be ignored + - `dtIgnore`: detections which should be ignored [T, D], + indicate which detections should be ignored + """ + # filter for max_detections highest scoring predictions to speed up computation + dt_ind = np.argsort(-pred_scores, kind="mergesort") + dt_ind = dt_ind[:max_detections] + + pred_boxes = pred_boxes[dt_ind] + pred_scores = pred_scores[dt_ind] + + # sort ignored ground truth to last positions + gt_ind = np.argsort(gt_ignore, kind="mergesort") + gt_boxes = gt_boxes[gt_ind] + gt_ignore = gt_ignore[gt_ind] + + # ious between sorted(!) predictions and ground truth + ious = iou_fn(pred_boxes, gt_boxes) # array sized (num_preds, num_gts) + + num_preds, num_gts = ious.shape[0], ious.shape[1] + gt_match = np.zeros((len(iou_thresholds), num_gts)) + dt_match = np.zeros((len(iou_thresholds), num_preds)) + dt_ignore = np.zeros((len(iou_thresholds), num_preds)) + + for tind, t in enumerate(iou_thresholds): + for dind, _d in enumerate(pred_boxes): # iterate detections starting from highest scoring one + # information about best match so far (m=-1 -> unmatched) + iou = min([t, 1 - 1e-10]) + m = -1 + + for gind, _g in enumerate(gt_boxes): # iterate ground truth + # if this gt already matched, continue + if gt_match[tind, gind] > 0: + continue + + # if dt matched to reg gt, and on ignore gt, stop + if m > -1 and gt_ignore[m] == 0 and gt_ignore[gind] == 1: + break + + # continue to next gt unless better match made + if ious[dind, gind] < iou: + continue + + # if match successful and best so far, store appropriately + iou = ious[dind, gind] + m = gind + + # if match made, store id of match for both dt and gt + if m == -1: + continue + else: + dt_ignore[tind, dind] = int(gt_ignore[m]) + dt_match[tind, dind] = 1 + gt_match[tind, m] = 1 + + # store results for given image and category + return { + "dtMatches": dt_match, # [T, D], where T = number of thresholds, D = number of detections + "gtMatches": gt_match, # [T, G], where T = number of thresholds, G = number of ground truth + "dtScores": pred_scores, # [D] detection scores + "gtIgnore": gt_ignore.reshape(-1), # [G] indicate whether ground truth should be ignored + "dtIgnore": dt_ignore, # [T, D], indicate which detections should be ignored + } diff --git a/tests/test_detection_coco_metrics.py b/tests/test_detection_coco_metrics.py new file mode 100644 index 0000000000..d5c8445f8e --- /dev/null +++ b/tests/test_detection_coco_metrics.py @@ -0,0 +1,66 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch + +from monai.apps.detection.metrics.coco import COCOMetric +from monai.apps.detection.metrics.matching import matching_batch +from monai.data.box_utils import box_iou + + +class TestCOCOMetrics(unittest.TestCase): + def test_coco_run(self): + coco_metric = COCOMetric(classes=["c0", "c1", "c2"], iou_list=[0.1], max_detection=[10]) + + num_images = 10 + + val_outputs_all = [] + val_targets_all = [] + for _ in range(num_images): + # randomly generate gt boxes and pred boxes + num_gt_boxes = random.randint(1, 3) + num_pred_boxes = random.randint(0, 3) + + box_start = torch.randint(3, (num_pred_boxes, 3)) + box_stop = box_start + torch.randint(1, 32, (num_pred_boxes, 3)) + boxes = torch.cat((box_start, box_stop), dim=1).to(torch.float16) + val_outputs_all.append( + { + "boxes": boxes, + "labels": torch.randint(3, (num_pred_boxes,)), + "scores": torch.randn((num_pred_boxes,)).absolute(), + } + ) + + box_start = torch.randint(3, (num_gt_boxes, 3)) + box_stop = box_start + torch.randint(1, 32, (num_gt_boxes, 3)) + boxes = torch.cat((box_start, box_stop), dim=1).to(torch.float16) + val_targets_all.append({"boxes": boxes, "labels": torch.randint(3, (num_gt_boxes,))}) + + results_metric = matching_batch( + iou_fn=box_iou, + iou_thresholds=coco_metric.iou_thresholds, + pred_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_outputs_all], + pred_classes=[val_data_i["labels"].numpy() for val_data_i in val_outputs_all], + pred_scores=[val_data_i["scores"].numpy() for val_data_i in val_outputs_all], + gt_boxes=[val_data_i["boxes"].numpy() for val_data_i in val_targets_all], + gt_classes=[val_data_i["labels"].numpy() for val_data_i in val_targets_all], + ) + val_epoch_metric_dict = coco_metric(results_metric)[0] + np.testing.assert_array_less([-0.01], [sum(val_epoch_metric_dict.values())]) + + +if __name__ == "__main__": + unittest.main()