diff --git a/docs/source/handlers.rst b/docs/source/handlers.rst index 0172529f40..189df18c43 100644 --- a/docs/source/handlers.rst +++ b/docs/source/handlers.rst @@ -41,6 +41,12 @@ Mean Dice metrics handler :members: +Mean IoU metric handler +----------------------- +.. autoclass:: MeanIoUHandler + :members: + + ROC AUC metrics handler ----------------------- .. autoclass:: ROCAUC diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py index cffbe46391..06a5ee6dbf 100644 --- a/monai/handlers/__init__.py +++ b/monai/handlers/__init__.py @@ -20,6 +20,7 @@ from .ignite_metric import IgniteMetric from .lr_schedule_handler import LrScheduleHandler from .mean_dice import MeanDice +from .mean_iou import MeanIoUHandler from .metric_logger import MetricLogger, MetricLoggerKeys from .metrics_saver import MetricsSaver from .mlflow_handler import MLFlowHandler diff --git a/monai/handlers/mean_iou.py b/monai/handlers/mean_iou.py new file mode 100644 index 0000000000..ee4602e6a7 --- /dev/null +++ b/monai/handlers/mean_iou.py @@ -0,0 +1,52 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Union + +from monai.handlers.ignite_metric import IgniteMetric +from monai.metrics import MeanIoU +from monai.utils import MetricReduction + + +class MeanIoUHandler(IgniteMetric): + """ + Computes IoU score metric from full size Tensor and collects average over batch, class-channels, iterations. + """ + + def __init__( + self, + include_background: bool = True, + reduction: Union[MetricReduction, str] = MetricReduction.MEAN, + output_transform: Callable = lambda x: x, + save_details: bool = True, + ) -> None: + """ + + Args: + include_background: whether to include iou computation on the first channel of the predicted output. + Defaults to True. + reduction: define the mode to reduce metrics, will only execute reduction on `not-nan` values, + available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``, + ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction. + output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then + construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or + lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`. + `engine.state` and `output_transform` inherit from the ignite concept: + https://pytorch.org/ignite/concepts.html#state, explanation and usage example are in the tutorial: + https://github.com/Project-MONAI/tutorials/blob/master/modules/batch_output_transform.ipynb. + save_details: whether to save metric computation details per image, for example: mean iou of every image. + default to True, will save to `engine.state.metric_details` dict with the metric name as key. + + See also: + :py:meth:`monai.metrics.meaniou.compute_meaniou` + """ + metric_fn = MeanIoU(include_background=include_background, reduction=reduction) + super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details) diff --git a/tests/min_tests.py b/tests/min_tests.py index 18e7b136fe..349fd2d275 100644 --- a/tests/min_tests.py +++ b/tests/min_tests.py @@ -68,6 +68,7 @@ def run_testsuit(): "test_handler_hausdorff_distance", "test_handler_lr_scheduler", "test_handler_mean_dice", + "test_handler_mean_iou", "test_handler_metrics_saver", "test_handler_metrics_saver_dist", "test_handler_mlflow", diff --git a/tests/test_handler_mean_iou.py b/tests/test_handler_mean_iou.py new file mode 100644 index 0000000000..9330efca99 --- /dev/null +++ b/tests/test_handler_mean_iou.py @@ -0,0 +1,72 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch +from ignite.engine import Engine, Events +from parameterized import parameterized + +from monai.handlers import MeanIoUHandler, from_engine + +TEST_CASE_1 = [{"include_background": True, "output_transform": from_engine(["pred", "label"])}, 0.75, (4, 2)] +TEST_CASE_2 = [{"include_background": False, "output_transform": from_engine(["pred", "label"])}, 2 / 3, (4, 1)] +TEST_CASE_3 = [ + {"reduction": "mean_channel", "output_transform": from_engine(["pred", "label"])}, + torch.Tensor([1.0, 0.0, 1.0, 1.0]), + (4, 2), +] + + +class TestHandlerMeanIoU(unittest.TestCase): + # TODO test multi node averaged iou + + @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3]) + def test_compute(self, input_params, expected_avg, details_shape): + iou_metric = MeanIoUHandler(**input_params) + # set up engine + + def _val_func(engine, batch): + pass + + engine = Engine(_val_func) + iou_metric.attach(engine=engine, name="mean_iou") + # test input a list of channel-first tensor + y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])] + y = torch.Tensor([[[0], [1]], [[0], [1]]]) + engine.state.output = {"pred": y_pred, "label": y} + engine.fire_event(Events.ITERATION_COMPLETED) + + y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])] + y = torch.Tensor([[[0], [1]], [[1], [0]]]) + engine.state.output = {"pred": y_pred, "label": y} + engine.fire_event(Events.ITERATION_COMPLETED) + + engine.fire_event(Events.EPOCH_COMPLETED) + torch.testing.assert_allclose(engine.state.metrics["mean_iou"], expected_avg) + self.assertTupleEqual(tuple(engine.state.metric_details["mean_iou"].shape), details_shape) + + @parameterized.expand([TEST_CASE_1, TEST_CASE_2]) + def test_shape_mismatch(self, input_params, _expected_avg, _details_shape): + iou_metric = MeanIoUHandler(**input_params) + with self.assertRaises((AssertionError, ValueError)): + y_pred = torch.Tensor([[0, 1], [1, 0]]) + y = torch.ones((3, 30)) + iou_metric.update([y_pred, y]) + + with self.assertRaises((AssertionError, ValueError)): + y_pred = torch.Tensor([[0, 1], [1, 0]]) + y = torch.ones((8, 30)) + iou_metric.update([y_pred, y]) + + +if __name__ == "__main__": + unittest.main()