From 4896db3b2ccd3e3f78764d55f5405c69f8c18f3d Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Tue, 18 Jan 2022 22:59:53 +0800 Subject: [PATCH 1/3] [DLMED] change to warning Signed-off-by: Nic Ma --- monai/metrics/rocauc.py | 4 +++- tests/test_compute_roc_auc.py | 17 +++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py index 341d4cba2f..c36bb507c9 100644 --- a/monai/metrics/rocauc.py +++ b/monai/metrics/rocauc.py @@ -9,6 +9,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import warnings from typing import Union, cast import numpy as np @@ -66,7 +67,8 @@ def _calculate(y_pred: torch.Tensor, y: torch.Tensor) -> float: if not (y.ndimension() == y_pred.ndimension() == 1 and len(y) == len(y_pred)): raise AssertionError("y and y_pred must be 1 dimension data with same length.") if not y.unique().equal(torch.tensor([0, 1], dtype=y.dtype, device=y.device)): - raise AssertionError("y values must be 0 or 1, can not be all 0 or all 1.") + warnings.warn("y values must be 0 or 1, can not be all 0 or all 1, skip AUC computation and return `Nan`.") + return float("nan") n = len(y) indices = y_pred.argsort() y = y[indices].cpu().numpy() diff --git a/tests/test_compute_roc_auc.py b/tests/test_compute_roc_auc.py index d06eed8740..592bf9459c 100644 --- a/tests/test_compute_roc_auc.py +++ b/tests/test_compute_roc_auc.py @@ -68,9 +68,20 @@ 0.62, ] +TEST_CASE_8 = [ + torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]), + torch.tensor([[0], [0], [0], [0]]), + True, + 2, + "macro", + float("nan"), +] + class TestComputeROCAUC(unittest.TestCase): - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) + @parameterized.expand( + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + ) def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value): y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)]) y_trans = Compose([ToTensor(), AsDiscrete(to_onehot=to_onehot)]) @@ -79,7 +90,9 @@ def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value): result = compute_roc_auc(y_pred=y_pred, y=y, average=average) np.testing.assert_allclose(expected_value, result, rtol=1e-5) - @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) + @parameterized.expand( + [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + ) def test_class_value(self, y_pred, y, softmax, to_onehot, average, expected_value): y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)]) y_trans = Compose([ToTensor(), AsDiscrete(to_onehot=to_onehot)]) From 2255080af45bf0120f10e5531e01b133aea06614 Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 19 Jan 2022 01:28:23 +0800 Subject: [PATCH 2/3] [DLMED] add more error message Signed-off-by: Nic Ma --- monai/metrics/rocauc.py | 12 ++++++++-- tests/test_compute_roc_auc.py | 44 +++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py index c36bb507c9..dfe9297295 100644 --- a/monai/metrics/rocauc.py +++ b/monai/metrics/rocauc.py @@ -66,9 +66,17 @@ def aggregate(self): # type: ignore def _calculate(y_pred: torch.Tensor, y: torch.Tensor) -> float: if not (y.ndimension() == y_pred.ndimension() == 1 and len(y) == len(y_pred)): raise AssertionError("y and y_pred must be 1 dimension data with same length.") - if not y.unique().equal(torch.tensor([0, 1], dtype=y.dtype, device=y.device)): - warnings.warn("y values must be 0 or 1, can not be all 0 or all 1, skip AUC computation and return `Nan`.") + y_unique = y.unique() + if y_unique.equal(torch.tensor([0], dtype=y.dtype, device=y.device)): + warnings.warn("y values can not be all 0, skip AUC computation and return `Nan`.") return float("nan") + if y_unique.equal(torch.tensor([1], dtype=y.dtype, device=y.device)): + warnings.warn("y values can not be all 1, skip AUC computation and return `Nan`.") + return float("nan") + if not y_unique.equal(torch.tensor([0, 1], dtype=y.dtype, device=y.device)): + warnings.warn("y values must be 0 or 1, skip AUC computation and return `Nan`.") + return float("nan") + n = len(y) indices = y_pred.argsort() y = y[indices].cpu().numpy() diff --git a/tests/test_compute_roc_auc.py b/tests/test_compute_roc_auc.py index 592bf9459c..887db08c7c 100644 --- a/tests/test_compute_roc_auc.py +++ b/tests/test_compute_roc_auc.py @@ -77,10 +77,39 @@ float("nan"), ] +TEST_CASE_9 = [ + torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]), + torch.tensor([[1], [1], [1], [1]]), + True, + 2, + "macro", + float("nan"), +] + +TEST_CASE_10 = [ + torch.tensor([[0.1, 0.9], [0.3, 1.4], [0.2, 0.1], [0.1, 0.5]]), + torch.tensor([[0, 0], [1, 1], [2, 2], [3, 3]]), + True, + None, + "macro", + float("nan"), +] + class TestComputeROCAUC(unittest.TestCase): @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + [ + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4, + TEST_CASE_5, + TEST_CASE_6, + TEST_CASE_7, + TEST_CASE_8, + TEST_CASE_9, + TEST_CASE_10, + ] ) def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value): y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)]) @@ -91,7 +120,18 @@ def test_value(self, y_pred, y, softmax, to_onehot, average, expected_value): np.testing.assert_allclose(expected_value, result, rtol=1e-5) @parameterized.expand( - [TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7, TEST_CASE_8] + [ + TEST_CASE_1, + TEST_CASE_2, + TEST_CASE_3, + TEST_CASE_4, + TEST_CASE_5, + TEST_CASE_6, + TEST_CASE_7, + TEST_CASE_8, + TEST_CASE_9, + TEST_CASE_10, + ] ) def test_class_value(self, y_pred, y, softmax, to_onehot, average, expected_value): y_pred_trans = Compose([ToTensor(), Activations(softmax=softmax)]) From 0b365805520cec603dbe57018c37a5babd764d8d Mon Sep 17 00:00:00 2001 From: Nic Ma Date: Wed, 19 Jan 2022 08:04:53 +0800 Subject: [PATCH 3/3] [DLMED] simplify the messages Signed-off-by: Nic Ma --- monai/metrics/rocauc.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/monai/metrics/rocauc.py b/monai/metrics/rocauc.py index dfe9297295..73f15534a9 100644 --- a/monai/metrics/rocauc.py +++ b/monai/metrics/rocauc.py @@ -67,14 +67,11 @@ def _calculate(y_pred: torch.Tensor, y: torch.Tensor) -> float: if not (y.ndimension() == y_pred.ndimension() == 1 and len(y) == len(y_pred)): raise AssertionError("y and y_pred must be 1 dimension data with same length.") y_unique = y.unique() - if y_unique.equal(torch.tensor([0], dtype=y.dtype, device=y.device)): - warnings.warn("y values can not be all 0, skip AUC computation and return `Nan`.") - return float("nan") - if y_unique.equal(torch.tensor([1], dtype=y.dtype, device=y.device)): - warnings.warn("y values can not be all 1, skip AUC computation and return `Nan`.") + if len(y_unique) == 1: + warnings.warn(f"y values can not be all {y_unique.item()}, skip AUC computation and return `Nan`.") return float("nan") if not y_unique.equal(torch.tensor([0, 1], dtype=y.dtype, device=y.device)): - warnings.warn("y values must be 0 or 1, skip AUC computation and return `Nan`.") + warnings.warn(f"y values must be 0 or 1, but in {y_unique.tolist()}, skip AUC computation and return `Nan`.") return float("nan") n = len(y)