4910 Add the mean iou handler

docs/source/handlers.rst

@@ -41,6 +41,12 @@ Mean Dice metrics handler
     :members:
 
 
+Mean IoU metric handler
+-----------------------
+.. autoclass:: MeanIoUHandler
+    :members:
+
+
 ROC AUC metrics handler
 -----------------------
 .. autoclass:: ROCAUC

monai/handlers/__init__.py

@@ -20,6 +20,7 @@
 from .ignite_metric import IgniteMetric
 from .lr_schedule_handler import LrScheduleHandler
 from .mean_dice import MeanDice
+from .mean_iou import MeanIoUHandler
 from .metric_logger import MetricLogger, MetricLoggerKeys
 from .metrics_saver import MetricsSaver
 from .mlflow_handler import MLFlowHandler

monai/handlers/mean_iou.py

@@ -0,0 +1,52 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Callable, Union
+
+from monai.handlers.ignite_metric import IgniteMetric
+from monai.metrics import MeanIoU
+from monai.utils import MetricReduction
+
+
+class MeanIoUHandler(IgniteMetric):
+    """
+    Computes IoU score metric from full size Tensor and collects average over batch, class-channels, iterations.
+    """
+
+    def __init__(
+        self,
+        include_background: bool = True,
+        reduction: Union[MetricReduction, str] = MetricReduction.MEAN,
+        output_transform: Callable = lambda x: x,
+        save_details: bool = True,
+    ) -> None:
+        """
+
+        Args:
+            include_background: whether to include iou computation on the first channel of the predicted output.
+                Defaults to True.
+            reduction: define the mode to reduce metrics, will only execute reduction on `not-nan` values,
+                available reduction modes: {``"none"``, ``"mean"``, ``"sum"``, ``"mean_batch"``, ``"sum_batch"``,
+                ``"mean_channel"``, ``"sum_channel"``}, default to ``"mean"``. if "none", will not do reduction.
+            output_transform: callable to extract `y_pred` and `y` from `ignite.engine.state.output` then
+                construct `(y_pred, y)` pair, where `y_pred` and `y` can be `batch-first` Tensors or
+                lists of `channel-first` Tensors. the form of `(y_pred, y)` is required by the `update()`.
+                `engine.state` and `output_transform` inherit from the ignite concept:
+                https://pytorch.org/ignite/concepts.html#state, explanation and usage example are in the tutorial:
+                https://github.com/Project-MONAI/tutorials/blob/master/modules/batch_output_transform.ipynb.
+            save_details: whether to save metric computation details per image, for example: mean iou of every image.
+                default to True, will save to `engine.state.metric_details` dict with the metric name as key.
+
+        See also:
+            :py:meth:`monai.metrics.meaniou.compute_meaniou`
+        """
+        metric_fn = MeanIoU(include_background=include_background, reduction=reduction)
+        super().__init__(metric_fn=metric_fn, output_transform=output_transform, save_details=save_details)

tests/min_tests.py

@@ -68,6 +68,7 @@ def run_testsuit():
         "test_handler_hausdorff_distance",
         "test_handler_lr_scheduler",
         "test_handler_mean_dice",
+        "test_handler_mean_iou",
         "test_handler_metrics_saver",
         "test_handler_metrics_saver_dist",
         "test_handler_mlflow",

tests/test_handler_mean_iou.py

@@ -0,0 +1,72 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import torch
+from ignite.engine import Engine, Events
+from parameterized import parameterized
+
+from monai.handlers import MeanIoUHandler, from_engine
+
+TEST_CASE_1 = [{"include_background": True, "output_transform": from_engine(["pred", "label"])}, 0.75, (4, 2)]
+TEST_CASE_2 = [{"include_background": False, "output_transform": from_engine(["pred", "label"])}, 2 / 3, (4, 1)]
+TEST_CASE_3 = [
+    {"reduction": "mean_channel", "output_transform": from_engine(["pred", "label"])},
+    torch.Tensor([1.0, 0.0, 1.0, 1.0]),
+    (4, 2),
+]
+
+
+class TestHandlerMeanIoU(unittest.TestCase):
+    # TODO test multi node averaged iou
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3])
+    def test_compute(self, input_params, expected_avg, details_shape):
+        iou_metric = MeanIoUHandler(**input_params)
+        # set up engine
+
+        def _val_func(engine, batch):
+            pass
+
+        engine = Engine(_val_func)
+        iou_metric.attach(engine=engine, name="mean_iou")
+        # test input a list of channel-first tensor
+        y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])]
+        y = torch.Tensor([[[0], [1]], [[0], [1]]])
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        y_pred = [torch.Tensor([[0], [1]]), torch.Tensor([[1], [0]])]
+        y = torch.Tensor([[[0], [1]], [[1], [0]]])
+        engine.state.output = {"pred": y_pred, "label": y}
+        engine.fire_event(Events.ITERATION_COMPLETED)
+
+        engine.fire_event(Events.EPOCH_COMPLETED)
+        torch.testing.assert_allclose(engine.state.metrics["mean_iou"], expected_avg)
+        self.assertTupleEqual(tuple(engine.state.metric_details["mean_iou"].shape), details_shape)
+
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    def test_shape_mismatch(self, input_params, _expected_avg, _details_shape):
+        iou_metric = MeanIoUHandler(**input_params)
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = torch.Tensor([[0, 1], [1, 0]])
+            y = torch.ones((3, 30))
+            iou_metric.update([y_pred, y])
+
+        with self.assertRaises((AssertionError, ValueError)):
+            y_pred = torch.Tensor([[0, 1], [1, 0]])
+            y = torch.ones((8, 30))
+            iou_metric.update([y_pred, y])
+
+
+if __name__ == "__main__":
+    unittest.main()