3686 Skip workflow run if data is empty or the specified epoch_length is 0

monai/engines/workflow.py

@@ -272,6 +272,13 @@ def run(self) -> None:
         Execute training, validation or evaluation based on Ignite Engine.
 
         """
+        if self.state.epoch_length == 0:
+            warnings.warn(
+                "`dataloader` is emply or the specified `epoch_length` is 0, skip the `run`."
+                " if running distributed training, the program may hang in `all-gather`, `all-reduce`, etc."
+                " because not all the ranks run the same computation logic."
+            )
+            return
         super().run(data=self.data_loader, max_epochs=self.state.max_epochs)
 
     def _iteration(self, engine: Engine, batchdata: Dict[str, torch.Tensor]):

tests/min_tests.py

@@ -154,6 +154,7 @@ def run_testsuit():
         "test_zoom",
         "test_zoom_affine",
         "test_zoomd",
+        "test_prepare_batch_default_dist",
     ]
     assert sorted(exclude_cases) == sorted(set(exclude_cases)), f"Duplicated items in {exclude_cases}"
 

tests/test_prepare_batch_default.py

@@ -49,6 +49,19 @@ def test_content(self):
         assert_allclose(output["image"], torch.tensor([1, 2], device=device))
         assert_allclose(output["label"], torch.tensor([3, 4], device=device))
 
+    def test_empty_data(self):
+        dataloader = []
+        evaluator = SupervisedEvaluator(
+            val_data_loader=dataloader,
+            device=torch.device("cpu"),
+            epoch_length=0,
+            network=TestNet(),
+            non_blocking=False,
+            prepare_batch=PrepareBatchDefault(),
+            decollate=False,
+        )
+        evaluator.run()
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_prepare_batch_default_dist.py

@@ -0,0 +1,73 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+import torch
+import torch.distributed as dist
+from parameterized import parameterized
+
+from monai.engines import PrepareBatchDefault, SupervisedEvaluator
+from tests.utils import DistCall, DistTestCase, assert_allclose
+
+TEST_CASE_1 = [
+    [
+        # data for rank 0, has 1 iteration
+        [{"image": torch.tensor([1, 1]), "label": torch.tensor([1, 0])}],
+        # data for rank 1, has 2 iterations
+        [
+            {"image": torch.tensor([1, 0]), "label": torch.tensor([1, 0])},
+            {"image": torch.tensor([1]), "label": torch.tensor([0])},
+        ],
+    ]
+]
+
+TEST_CASE_2 = [
+    [
+        # data for rank 0
+        [{"image": torch.tensor([0, 1, 1, 0, 1]), "label": torch.tensor([1, 1, 0, 0, 1])}],
+        # data for rank 1
+        [],
+    ]
+]
+
+
+class TestNet(torch.nn.Module):
+    def forward(self, x: torch.Tensor):
+        return x
+
+
+class DistributedPrepareBatchDefault(DistTestCase):
+    @parameterized.expand([TEST_CASE_1, TEST_CASE_2])
+    @DistCall(nnodes=1, nproc_per_node=2, node_rank=0)
+    def test_compute(self, dataloaders):
+        device = torch.device(f"cuda:{dist.get_rank()}" if torch.cuda.is_available() else "cpu")
+        dataloader = dataloaders[dist.get_rank()]
+        # set up engine
+        evaluator = SupervisedEvaluator(
+            device=device,
+            val_data_loader=dataloader,
+            epoch_length=len(dataloader),
+            network=TestNet(),
+            non_blocking=False,
+            prepare_batch=PrepareBatchDefault(),
+            decollate=False,
+        )
+        evaluator.run()
+        output = evaluator.state.output
+        if len(dataloader) > 0:
+            assert_allclose(output["image"], dataloader[-1]["image"].to(device=device))
+            assert_allclose(output["label"], dataloader[-1]["label"].to(device=device))
+
+
+if __name__ == "__main__":
+    unittest.main()