From 695bc74c2e021b490650331069b767f31868732b Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 24 Oct 2022 13:55:51 +0000
Subject: [PATCH 01/51] add experiment/run name to mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 664a1c8730..9d18cbd4b7 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -86,6 +86,8 @@ def __init__(
         global_epoch_transform: Callable = lambda x: x,
         state_attributes: Optional[Sequence[str]] = None,
         tag_name: str = DEFAULT_TAG,
+        experiment_name: str = "default_experiment",
+        run_name: str = "test_run",
     ) -> None:
         if tracking_uri is not None:
             mlflow.set_tracking_uri(tracking_uri)
@@ -98,6 +100,8 @@ def __init__(
         self.global_epoch_transform = global_epoch_transform
         self.state_attributes = state_attributes
         self.tag_name = tag_name
+        self.experiment_name = experiment_name
+        self.run_name = run_name
 
     def attach(self, engine: Engine) -> None:
         """
@@ -119,8 +123,9 @@ def start(self) -> None:
         Check MLFlow status and start if not active.
 
         """
+        mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            mlflow.start_run()
+            mlflow.start_run(run_name=self.run_name)
 
     def close(self) -> None:
         """

From b88a089653cf9df32f3cb43bdb5a77073893129a Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 25 Oct 2022 09:10:12 +0000
Subject: [PATCH 02/51] add apis to mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 40 +++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 9d18cbd4b7..1bb77c71ee 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import TYPE_CHECKING, Any, Callable, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence
 
 import torch
 
@@ -70,6 +70,13 @@ class MLFlowHandler:
         state_attributes: expected attributes from `engine.state`, if provided, will extract them
             when epoch completed.
         tag_name: when iteration output is a scalar, `tag_name` is used to track, defaults to `'Loss'`.
+        experiment_name: name for an experiment, defaults to `default_experiment`.
+        run_name: name for run in an experiment, defaults to `test_run`.
+        experiment_param: a dict recording parameters which will not change through whole experiment,
+            like torch version, cuda version and so on.
+        artifacts: paths to images that need to be recorded after a whole run.
+        optimizer_param_names: parameters' name in optimizer that need to be record during runing,
+            defaults to `["lr"]`.
 
     For more details of MLFlow usage, please refer to: https://mlflow.org/docs/latest/index.html.
 
@@ -88,6 +95,9 @@ def __init__(
         tag_name: str = DEFAULT_TAG,
         experiment_name: str = "default_experiment",
         run_name: str = "test_run",
+        experiment_param: Optional[Dict] = None,
+        artifacts: Optional[Sequence[str]] = None,
+        optimizer_param_names: Sequence[str] = ["lr"],
     ) -> None:
         if tracking_uri is not None:
             mlflow.set_tracking_uri(tracking_uri)
@@ -102,6 +112,9 @@ def __init__(
         self.tag_name = tag_name
         self.experiment_name = experiment_name
         self.run_name = run_name
+        self.experiment_param = experiment_param
+        self.artifacts = artifacts
+        self.optimizer_param_names = optimizer_param_names
 
     def attach(self, engine: Engine) -> None:
         """
@@ -117,6 +130,8 @@ def attach(self, engine: Engine) -> None:
             engine.add_event_handler(Events.ITERATION_COMPLETED, self.iteration_completed)
         if self.epoch_log and not engine.has_event_handler(self.epoch_completed, Events.EPOCH_COMPLETED):
             engine.add_event_handler(Events.EPOCH_COMPLETED, self.epoch_completed)
+        if not engine.has_event_handler(self.complete, Events.COMPLETED):
+            engine.add_event_handler(Events.EPOCH_COMPLETED, self.complete)
 
     def start(self) -> None:
         """
@@ -127,6 +142,16 @@ def start(self) -> None:
         if mlflow.active_run() is None:
             mlflow.start_run(run_name=self.run_name)
 
+        if self.experiment_param:
+            mlflow.log_params(self.experiment_param)
+
+    def complete(self) -> None:
+        """
+        Handler for train or validation/evaluation completed Event.
+        """
+        if self.artifacts:
+            mlflow.log_artifacts(self.artifacts)
+
     def close(self) -> None:
         """
         Stop current running logger of MLFlow.
@@ -202,3 +227,16 @@ def _default_iteration_log(self, engine: Engine) -> None:
             loss = {self.tag_name: loss.item() if isinstance(loss, torch.Tensor) else loss}
 
         mlflow.log_metrics(loss, step=engine.state.iteration)
+
+        # If there is optimizer attr in engine, then record parameters specified in init function.
+        try:
+            cur_optimizer = engine.optimizer
+            for param_name in self.optimizer_param_names:
+                params = {
+                    f"{param_name} group_{i}": float(param_group[param_name])
+                    for i, param_group in enumerate(cur_optimizer.param_groups)
+                }
+                mlflow.log_metrics(params, step=engine.state.iteration)
+
+        except AttributeError:
+            pass

From cd8ef5a1365630fa1d4727d5f362c9bf7a9d9e59 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 27 Oct 2022 07:08:47 +0000
Subject: [PATCH 03/51] add some default record in mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 1bb77c71ee..5d8e57883e 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -115,6 +115,7 @@ def __init__(
         self.experiment_param = experiment_param
         self.artifacts = artifacts
         self.optimizer_param_names = optimizer_param_names
+        self.default_attr_name = ["seed", "max_epochs", "epoch_length"]
 
     def attach(self, engine: Engine) -> None:
         """
@@ -133,7 +134,7 @@ def attach(self, engine: Engine) -> None:
         if not engine.has_event_handler(self.complete, Events.COMPLETED):
             engine.add_event_handler(Events.EPOCH_COMPLETED, self.complete)
 
-    def start(self) -> None:
+    def start(self, engine: Engine) -> None:
         """
         Check MLFlow status and start if not active.
 
@@ -145,6 +146,27 @@ def start(self) -> None:
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)
 
+        attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
+        mlflow.log_params(attrs)
+
+        try:
+            network_string = str(type(engine.network))
+            mlflow.log_param(key="network", value=network_string)
+        except AttributeError:
+            pass
+
+        try:
+            optimizer_string = str(type(engine.optimizer))
+            loss_string = str(type(engine.loss_function))
+            mlflow.log_params(
+                {
+                    "optimizer": optimizer_string,
+                    "loss_function": loss_string,
+                }
+            )
+        except AttributeError:
+            pass
+
     def complete(self) -> None:
         """
         Handler for train or validation/evaluation completed Event.

From 6b68da0925576c3f1088bcdd2a4a76d752a76ecb Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 27 Oct 2022 15:32:50 +0000
Subject: [PATCH 04/51] add val metrics to mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 53 +++++++++++++++++++-------------
 1 file changed, 32 insertions(+), 21 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 5d8e57883e..b43451817a 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -9,11 +9,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import time
+from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence
 
 import torch
 
 from monai.config import IgniteInfo
+from monai.handlers.validation_handler import ValidationHandler
 from monai.utils import min_version, optional_import
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
@@ -94,9 +97,9 @@ def __init__(
         state_attributes: Optional[Sequence[str]] = None,
         tag_name: str = DEFAULT_TAG,
         experiment_name: str = "default_experiment",
-        run_name: str = "test_run",
+        run_name: Optional[str] = None,
         experiment_param: Optional[Dict] = None,
-        artifacts: Optional[Sequence[str]] = None,
+        artifacts: Optional[Sequence[Path]] = None,
         optimizer_param_names: Sequence[str] = ["lr"],
     ) -> None:
         if tracking_uri is not None:
@@ -117,6 +120,12 @@ def __init__(
         self.optimizer_param_names = optimizer_param_names
         self.default_attr_name = ["seed", "max_epochs", "epoch_length"]
 
+    def _try_log_param(self, engine: Engine, attr: str):
+        engine_attr = getattr(engine, attr, None)
+        if engine_attr:
+            attr_type_string = str(type(engine_attr))
+            mlflow.log_param(key=attr, value=attr_type_string)
+
     def attach(self, engine: Engine) -> None:
         """
         Register a set of Ignite Event-Handlers to a specified Ignite engine.
@@ -141,7 +150,12 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            mlflow.start_run(run_name=self.run_name)
+            if self.run_name:
+                mlflow.start_run(run_name=self.run_name)
+            else:
+                cur_time = time.strftime("%Y%m%d_%H%M%S")
+                run_name = "run_" + cur_time
+                mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)
@@ -149,23 +163,10 @@ def start(self, engine: Engine) -> None:
         attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
         mlflow.log_params(attrs)
 
-        try:
-            network_string = str(type(engine.network))
-            mlflow.log_param(key="network", value=network_string)
-        except AttributeError:
-            pass
-
-        try:
-            optimizer_string = str(type(engine.optimizer))
-            loss_string = str(type(engine.loss_function))
-            mlflow.log_params(
-                {
-                    "optimizer": optimizer_string,
-                    "loss_function": loss_string,
-                }
-            )
-        except AttributeError:
-            pass
+        self._try_log_param(engine, "network")
+        self._try_log_param(engine, "device")
+        self._try_log_param(engine, "optimizer")
+        self._try_log_param(engine, "loss_function")
 
     def complete(self) -> None:
         """
@@ -226,6 +227,16 @@ def _default_epoch_log(self, engine: Engine) -> None:
         current_epoch = self.global_epoch_transform(engine.state.epoch)
         mlflow.log_metrics(log_dict, step=current_epoch)
 
+        events = engine._event_handlers
+
+        for e in events:
+            for handler, _, _ in engine._event_handlers[e]:
+                if isinstance(handler, ValidationHandler):
+                    evaluator_state = getattr(handler.validator, "state", None)
+                    if evaluator_state:
+                        handler_metrics_dict = evaluator_state.metrics
+                        mlflow.log_metrics(handler_metrics_dict, step=current_epoch)
+
         if self.state_attributes is not None:
             attrs = {attr: getattr(engine.state, attr, None) for attr in self.state_attributes}
             mlflow.log_metrics(attrs, step=current_epoch)
@@ -252,7 +263,7 @@ def _default_iteration_log(self, engine: Engine) -> None:
 
         # If there is optimizer attr in engine, then record parameters specified in init function.
         try:
-            cur_optimizer = engine.optimizer
+            cur_optimizer = engine.optimizer  # type: ignore
             for param_name in self.optimizer_param_names:
                 params = {
                     f"{param_name} group_{i}": float(param_group[param_name])

From 8e89edbf309a71cc87e0760719f262a7da74fc12 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 28 Oct 2022 05:43:49 +0000
Subject: [PATCH 05/51] update iteration log in case mlflow handler attaching
 to non trainable engine

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index b43451817a..3520b8ffa8 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -16,6 +16,7 @@
 import torch
 
 from monai.config import IgniteInfo
+from monai.engines import Trainer
 from monai.handlers.validation_handler import ValidationHandler
 from monai.utils import min_version, optional_import
 
@@ -259,7 +260,8 @@ def _default_iteration_log(self, engine: Engine) -> None:
         if not isinstance(loss, dict):
             loss = {self.tag_name: loss.item() if isinstance(loss, torch.Tensor) else loss}
 
-        mlflow.log_metrics(loss, step=engine.state.iteration)
+        if isinstance(engine, Trainer):
+            mlflow.log_metrics(loss, step=engine.state.iteration)
 
         # If there is optimizer attr in engine, then record parameters specified in init function.
         try:

From abeb28962f56493dbb569c7431037556da9a1797 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 2 Nov 2022 08:24:38 +0000
Subject: [PATCH 06/51] add check for existed params

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 43 ++++++++++++++++++++++++++++----
 1 file changed, 38 insertions(+), 5 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 3520b8ffa8..4529f06ce4 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -10,6 +10,7 @@
 # limitations under the License.
 
 import time
+import os
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence
 
@@ -120,12 +121,27 @@ def __init__(
         self.artifacts = artifacts
         self.optimizer_param_names = optimizer_param_names
         self.default_attr_name = ["seed", "max_epochs", "epoch_length"]
+        self.client = mlflow.MlflowClient()
 
     def _try_log_param(self, engine: Engine, attr: str):
         engine_attr = getattr(engine, attr, None)
         if engine_attr:
             attr_type_string = str(type(engine_attr))
             mlflow.log_param(key=attr, value=attr_type_string)
+    
+    def _is_param_exists(self, param_name:str):
+        cur_run = mlflow.active_run()
+        log_data = self.client.get_run(cur_run.info.run_id).data
+        param_dict = log_data.params
+        if param_name in param_dict:
+            return True
+        else:
+            return False
+    def _delete_exist_param_in_dict(self, param_dict):
+        key_list = param_dict.keys()
+        for key in key_list:
+            if self._is_param_exists(key):
+                del param_dict[key]
 
     def attach(self, engine: Engine) -> None:
         """
@@ -162,19 +178,36 @@ def start(self, engine: Engine) -> None:
             mlflow.log_params(self.experiment_param)
 
         attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
+        self._delete_exist_param_in_dict(attrs)
         mlflow.log_params(attrs)
 
-        self._try_log_param(engine, "network")
-        self._try_log_param(engine, "device")
-        self._try_log_param(engine, "optimizer")
-        self._try_log_param(engine, "loss_function")
+        default_log_param_list = ["network", "device", "optimizer", "loss_function"]
+        for param_name in default_log_param_list:
+            if self._is_param_exists(param_name):
+                continue
+            self._try_log_param(engine, param_name)
+
+    def _parse_artifacts(self):
+        artifact_list = []
+        for path_name in self.artifacts:
+            if os.path.isfile(path_name):
+                artifact_list.append(path_name)
+            else:
+                file_list = []
+                for root, _, filenames in os.walk(path_name)
+                    for filename in filenames:
+                        file_path = os.path.join(root, filename)
+                        file_list.append(file_path)
+                artifact_list.extend(file_list)
+        return artifact_list
 
     def complete(self) -> None:
         """
         Handler for train or validation/evaluation completed Event.
         """
         if self.artifacts:
-            mlflow.log_artifacts(self.artifacts)
+            artifact_list = self._parse_artifacts()
+            mlflow.log_artifacts(artifact_list)
 
     def close(self) -> None:
         """

From 1414c690510c80c24062c4484698e1d1b2de2b21 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 2 Nov 2022 16:16:41 +0000
Subject: [PATCH 07/51] update mlflow handler artifact log

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 4529f06ce4..9a31859c85 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -128,8 +128,8 @@ def _try_log_param(self, engine: Engine, attr: str):
         if engine_attr:
             attr_type_string = str(type(engine_attr))
             mlflow.log_param(key=attr, value=attr_type_string)
-    
-    def _is_param_exists(self, param_name:str):
+
+    def _is_param_exists(self, param_name: str):
         cur_run = mlflow.active_run()
         log_data = self.client.get_run(cur_run.info.run_id).data
         param_dict = log_data.params
@@ -137,8 +137,9 @@ def _is_param_exists(self, param_name:str):
             return True
         else:
             return False
+
     def _delete_exist_param_in_dict(self, param_dict):
-        key_list = param_dict.keys()
+        key_list = [x for x in param_dict.keys()]
         for key in key_list:
             if self._is_param_exists(key):
                 del param_dict[key]
@@ -193,12 +194,10 @@ def _parse_artifacts(self):
             if os.path.isfile(path_name):
                 artifact_list.append(path_name)
             else:
-                file_list = []
-                for root, _, filenames in os.walk(path_name)
+                for root, _, filenames in os.walk(path_name):
                     for filename in filenames:
                         file_path = os.path.join(root, filename)
-                        file_list.append(file_path)
-                artifact_list.extend(file_list)
+                        artifact_list.append(file_path)
         return artifact_list
 
     def complete(self) -> None:
@@ -207,7 +206,8 @@ def complete(self) -> None:
         """
         if self.artifacts:
             artifact_list = self._parse_artifacts()
-            mlflow.log_artifacts(artifact_list)
+            for artifact in artifact_list:
+                mlflow.log_artifact(artifact)
 
     def close(self) -> None:
         """

From 34e3a8ac087ae3c6ecb35a91bf97c261593db00c Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 15 Nov 2022 14:52:33 +0000
Subject: [PATCH 08/51] change mlflow run name according to input parameter.

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 1e0cee118b..c973357fa1 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -173,12 +173,8 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            if self.run_name:
-                mlflow.start_run(run_name=self.run_name)
-            else:
-                cur_time = time.strftime("%Y%m%d_%H%M%S")
-                run_name = "run_" + cur_time
-                mlflow.start_run(run_name=run_name)
+            run_name = f"run_{time.strftime("%Y%m%d_%H%M%S")}" if self.run_name is None else self.run_name
+            mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)

From a92eb198ff3122f6772058104c9dcd7e17dd08fe Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 05:00:29 +0000
Subject: [PATCH 09/51] format code

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 3ec886f658..1ac0944639 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -9,8 +9,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import time
 import os
+import time
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence
 

From 95227f731e17a739a95c78b2d7b3c0daf7e3548c Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 06:26:40 +0000
Subject: [PATCH 10/51] fix codeformat problem

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 1ac0944639..091d2108e5 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -170,7 +170,7 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            run_name = f"run_{time.strftime("%Y%m%d_%H%M%S")}" if self.run_name is None else self.run_name
+            run_name = f"run_{time.strftime('%Y%m%d_%H%M%S')}" if self.run_name is None else self.run_name
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:

From 5d4475f488050864d0b748a8a1c24804f2e7b142 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 07:15:29 +0000
Subject: [PATCH 11/51] update the code comments and handler for validator

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 51 ++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 19 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 091d2108e5..b777b05afc 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -78,7 +78,7 @@ class MLFlowHandler:
             when epoch completed.
         tag_name: when iteration output is a scalar, `tag_name` is used to track, defaults to `'Loss'`.
         experiment_name: name for an experiment, defaults to `default_experiment`.
-        run_name: name for run in an experiment, defaults to `test_run`.
+        run_name: name for run in an experiment.
         experiment_param: a dict recording parameters which will not change through whole experiment,
             like torch version, cuda version and so on.
         artifacts: paths to images that need to be recorded after a whole run.
@@ -89,6 +89,9 @@ class MLFlowHandler:
 
     """
 
+    default_log_param_list = ["network", "device", "optimizer", "loss_function"]
+    default_attr_name = ["seed", "max_epochs", "epoch_length"]
+
     def __init__(
         self,
         tracking_uri: Optional[str] = None,
@@ -122,16 +125,31 @@ def __init__(
         self.experiment_param = experiment_param
         self.artifacts = artifacts
         self.optimizer_param_names = optimizer_param_names
-        self.default_attr_name = ["seed", "max_epochs", "epoch_length"]
         self.client = mlflow.MlflowClient()
 
-    def _try_log_param(self, engine: Engine, attr: str):
+    def _try_log_param(self, engine: Engine, attr: str) -> None:
+        """
+        Log parameter to mlflow if it exists in given engine.
+
+        Args:
+            engine: A ignite engine.
+            attr: Attribute that needs to be logged by mlflow.
+        """
         engine_attr = getattr(engine, attr, None)
         if engine_attr:
             attr_type_string = str(type(engine_attr))
             mlflow.log_param(key=attr, value=attr_type_string)
 
-    def _is_param_exists(self, param_name: str):
+    def _is_param_exists(self, param_name: str) -> bool:
+        """
+        Check whether a parameter has already been logged in current mlflow run.
+
+        Args:
+            param_name: name of parameter to check in mlflow run.
+
+        Return:
+            If the given parameter already was logged to mlflow.
+        """
         cur_run = mlflow.active_run()
         log_data = self.client.get_run(cur_run.info.run_id).data
         param_dict = log_data.params
@@ -141,7 +159,13 @@ def _is_param_exists(self, param_name: str):
             return False
 
     def _delete_exist_param_in_dict(self, param_dict):
-        key_list = [x for x in param_dict.keys()]
+        """
+        Delete parameters in given dict, if they are already logged by current mlflow run.
+
+        Args:
+            param_dict: parameter dict to be logged to mlflow.
+        """
+        key_list = list(param_dict.keys())
         for key in key_list:
             if self._is_param_exists(key):
                 del param_dict[key]
@@ -161,7 +185,7 @@ def attach(self, engine: Engine) -> None:
         if self.epoch_log and not engine.has_event_handler(self.epoch_completed, Events.EPOCH_COMPLETED):
             engine.add_event_handler(Events.EPOCH_COMPLETED, self.epoch_completed)
         if not engine.has_event_handler(self.complete, Events.COMPLETED):
-            engine.add_event_handler(Events.EPOCH_COMPLETED, self.complete)
+            engine.add_event_handler(Events.COMPLETED, self.complete)
 
     def start(self, engine: Engine) -> None:
         """
@@ -176,12 +200,11 @@ def start(self, engine: Engine) -> None:
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)
 
-        attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
+        attrs = {attr: getattr(engine.state, attr, None) for attr in cls.default_attr_name}
         self._delete_exist_param_in_dict(attrs)
         mlflow.log_params(attrs)
 
-        default_log_param_list = ["network", "device", "optimizer", "loss_function"]
-        for param_name in default_log_param_list:
+        for param_name in cls.default_log_param_list:
             if self._is_param_exists(param_name):
                 continue
             self._try_log_param(engine, param_name)
@@ -259,16 +282,6 @@ def _default_epoch_log(self, engine: Engine) -> None:
         current_epoch = self.global_epoch_transform(engine.state.epoch)
         mlflow.log_metrics(log_dict, step=current_epoch)
 
-        events = engine._event_handlers
-
-        for e in events:
-            for handler, _, _ in engine._event_handlers[e]:
-                if isinstance(handler, ValidationHandler):
-                    evaluator_state = getattr(handler.validator, "state", None)
-                    if evaluator_state:
-                        handler_metrics_dict = evaluator_state.metrics
-                        mlflow.log_metrics(handler_metrics_dict, step=current_epoch)
-
         if self.state_attributes is not None:
             attrs = {attr: getattr(engine.state, attr, None) for attr in self.state_attributes}
             mlflow.log_metrics(attrs, step=current_epoch)

From e8760d7e1fb35db33c7e38f45c3913abc35e148d Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 07:19:21 +0000
Subject: [PATCH 12/51] update some type hints and comments

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index b777b05afc..63ec37cf1a 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -158,7 +158,7 @@ def _is_param_exists(self, param_name: str) -> bool:
         else:
             return False
 
-    def _delete_exist_param_in_dict(self, param_dict):
+    def _delete_exist_param_in_dict(self, param_dict: Dict) -> None:
         """
         Delete parameters in given dict, if they are already logged by current mlflow run.
 
@@ -210,6 +210,10 @@ def start(self, engine: Engine) -> None:
             self._try_log_param(engine, param_name)
 
     def _parse_artifacts(self):
+        """
+        Log artifacts to mlflow. Given a path, all files in the path will be logged recursively.
+        Given a file, it will be logged to mlflow.
+        """
         artifact_list = []
         for path_name in self.artifacts:
             if os.path.isfile(path_name):

From 7fe82699fcd9ded475d80e9e17140b9db26dd503 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 09:29:24 +0000
Subject: [PATCH 13/51] update default mlflow tracking with bundle info and
 multi gpu info

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/__init__.py         |  2 +-
 monai/bundle/scripts.py          | 10 ++++++++++
 monai/bundle/utils.py            |  3 ++-
 monai/handlers/mlflow_handler.py |  5 +++--
 4 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py
index 38cbb2c264..0ebe9ebe7e 100644
--- a/monai/bundle/__init__.py
+++ b/monai/bundle/__init__.py
@@ -25,4 +25,4 @@
     verify_metadata,
     verify_net_in_out,
 )
-from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config
+from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config, DEFAULT_MLFLOW_SETTINGS
diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index fc0d2a09b5..a4360ef764 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -469,6 +469,15 @@ def get_bundle_info(
     return bundle_info[version]
 
 
+def _add_config_to_mlflow_handler(parser: ConfigParser, handler_config: dict) -> None:
+    """
+    Add config info to mlflow handler.
+    """
+    if "MLFlowHandler" == handler_config["_target_"]:
+        config_json_file = json.dumps(parser.config)
+        handler_config["experiment_param"] = json.dumps({"bundle_info": config_json_file})
+
+
 def patch_bundle_tracking(parser: ConfigParser, settings: dict):
     """
     Patch the loaded bundle config with a new handler logic to enable experiment tracking features.
@@ -487,6 +496,7 @@ def patch_bundle_tracking(parser: ConfigParser, settings: dict):
                 if handlers is None:
                     engine["train_handlers" if k == "trainer" else "val_handlers"] = [handler_config]
                 else:
+                    _add_config_to_mlflow_handler(parser=parser, handler_config=handler_config)
                     handlers.append(handler_config)
 
 
diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 42d38bfeea..a5f45dd48b 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -19,7 +19,7 @@
 
 yaml, _ = optional_import("yaml")
 
-__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY"]
+__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS"]
 
 ID_REF_KEY = "@"  # start of a reference to a ConfigItem
 ID_SEP_KEY = "#"  # separator for the ID of a ConfigItem
@@ -107,6 +107,7 @@
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
+            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
             "tracking_uri": "$@output_dir + '/mlflow'",
             "iteration_log": True,
             "epoch_log": True,
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 63ec37cf1a..fe9e5f00a4 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -197,14 +197,15 @@ def start(self, engine: Engine) -> None:
             run_name = f"run_{time.strftime('%Y%m%d_%H%M%S')}" if self.run_name is None else self.run_name
             mlflow.start_run(run_name=run_name)
 
+        self._delete_exist_param_in_dict(self.experiment_param)
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)
 
-        attrs = {attr: getattr(engine.state, attr, None) for attr in cls.default_attr_name}
+        attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
         self._delete_exist_param_in_dict(attrs)
         mlflow.log_params(attrs)
 
-        for param_name in cls.default_log_param_list:
+        for param_name in self.default_log_param_list:
             if self._is_param_exists(param_name):
                 continue
             self._try_log_param(engine, param_name)

From 39f40b6f832eff1c7a7d25c35b20d830f7db14b9 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 09:47:29 +0000
Subject: [PATCH 14/51] fix the log default parameter bug

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/__init__.py         | 2 +-
 monai/handlers/mlflow_handler.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py
index 0ebe9ebe7e..f3a91cb89f 100644
--- a/monai/bundle/__init__.py
+++ b/monai/bundle/__init__.py
@@ -25,4 +25,4 @@
     verify_metadata,
     verify_net_in_out,
 )
-from .utils import EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config, DEFAULT_MLFLOW_SETTINGS
+from .utils import DEFAULT_MLFLOW_SETTINGS, EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index fe9e5f00a4..09e04eeedc 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -18,7 +18,6 @@
 
 from monai.config import IgniteInfo
 from monai.engines import Trainer
-from monai.handlers.validation_handler import ValidationHandler
 from monai.utils import min_version, optional_import
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
@@ -197,9 +196,11 @@ def start(self, engine: Engine) -> None:
             run_name = f"run_{time.strftime('%Y%m%d_%H%M%S')}" if self.run_name is None else self.run_name
             mlflow.start_run(run_name=run_name)
 
-        self._delete_exist_param_in_dict(self.experiment_param)
         if self.experiment_param:
-            mlflow.log_params(self.experiment_param)
+            # avoid to recording same parameters in the same run
+            self._delete_exist_param_in_dict(self.experiment_param)
+            if self.experiment_param:
+                mlflow.log_params(self.experiment_param)
 
         attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
         self._delete_exist_param_in_dict(attrs)

From e3d2cb3b183dcebf10b28ce109502871799c0986 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 10:21:49 +0000
Subject: [PATCH 15/51] add disable for multi gpu in validator and evaluator

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index a5f45dd48b..ef81abd199 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -115,9 +115,19 @@
             "output_transform": "$monai.handlers.from_engine(['loss'], first=True)",
         },
         # MLFlowHandler config for the validator
-        "validator": {"_target_": "MLFlowHandler", "tracking_uri": "$@output_dir + '/mlflow'", "iteration_log": False},
+        "validator": {
+            "_target_": "MLFlowHandler",
+            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "tracking_uri": "$@output_dir + '/mlflow'",
+            "iteration_log": False,
+        },
         # MLFlowHandler config for the evaluator
-        "evaluator": {"_target_": "MLFlowHandler", "tracking_uri": "$@output_dir + '/mlflow'", "iteration_log": False},
+        "evaluator": {
+            "_target_": "MLFlowHandler",
+            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "tracking_uri": "$@output_dir + '/mlflow'",
+            "iteration_log": False,
+        },
     },
 }
 

From f98e29ba10d15f4359a96e9e50cf2ab0cf3e9e28 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 13:58:57 +0000
Subject: [PATCH 16/51] fix the parameter bug and code format problem

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/scripts.py          | 5 +++--
 monai/bundle/utils.py            | 6 +++---
 monai/handlers/mlflow_handler.py | 4 ++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index a4360ef764..6877526d54 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -474,8 +474,9 @@ def _add_config_to_mlflow_handler(parser: ConfigParser, handler_config: dict) ->
     Add config info to mlflow handler.
     """
     if "MLFlowHandler" == handler_config["_target_"]:
-        config_json_file = json.dumps(parser.config)
-        handler_config["experiment_param"] = json.dumps({"bundle_info": config_json_file})
+        config_str = json.dumps(parser.config)
+        bundle_info_str = json.dumps({'bundle_info':config_str})
+        handler_config["experiment_param"] = "$" + bundle_info_str
 
 
 def patch_bundle_tracking(parser: ConfigParser, settings: dict):
diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index ef81abd199..a0f35fd027 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -107,7 +107,7 @@
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
-            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
             "tracking_uri": "$@output_dir + '/mlflow'",
             "iteration_log": True,
             "epoch_log": True,
@@ -117,14 +117,14 @@
         # MLFlowHandler config for the validator
         "validator": {
             "_target_": "MLFlowHandler",
-            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
             "tracking_uri": "$@output_dir + '/mlflow'",
             "iteration_log": False,
         },
         # MLFlowHandler config for the evaluator
         "evaluator": {
             "_target_": "MLFlowHandler",
-            "_disable_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
             "tracking_uri": "$@output_dir + '/mlflow'",
             "iteration_log": False,
         },
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 09e04eeedc..3db81542e3 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -12,7 +12,7 @@
 import os
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Tuple
 
 import torch
 
@@ -106,7 +106,7 @@ def __init__(
         run_name: Optional[str] = None,
         experiment_param: Optional[Dict] = None,
         artifacts: Optional[Sequence[Path]] = None,
-        optimizer_param_names: Sequence[str] = ["lr"],
+        optimizer_param_names: Tuple[str] = ("lr"),
     ) -> None:
         if tracking_uri is not None:
             mlflow.set_tracking_uri(tracking_uri)

From 38524b2f842d00d83331a51b514260c1dfdaef1e Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Tue, 22 Nov 2022 14:03:51 +0000
Subject: [PATCH 17/51] fix the code format problem

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/scripts.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index 6877526d54..9e54825a15 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -475,7 +475,7 @@ def _add_config_to_mlflow_handler(parser: ConfigParser, handler_config: dict) ->
     """
     if "MLFlowHandler" == handler_config["_target_"]:
         config_str = json.dumps(parser.config)
-        bundle_info_str = json.dumps({'bundle_info':config_str})
+        bundle_info_str = json.dumps({"bundle_info": config_str})
         handler_config["experiment_param"] = "$" + bundle_info_str
 
 

From 0214e1c42beba5be622da34e71e3f1c260ee700a Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 23 Nov 2022 08:02:01 +0000
Subject: [PATCH 18/51] remove the bundle config tracking code

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/scripts.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index 9e54825a15..49576556d1 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -468,17 +468,6 @@ def get_bundle_info(
 
     return bundle_info[version]
 
-
-def _add_config_to_mlflow_handler(parser: ConfigParser, handler_config: dict) -> None:
-    """
-    Add config info to mlflow handler.
-    """
-    if "MLFlowHandler" == handler_config["_target_"]:
-        config_str = json.dumps(parser.config)
-        bundle_info_str = json.dumps({"bundle_info": config_str})
-        handler_config["experiment_param"] = "$" + bundle_info_str
-
-
 def patch_bundle_tracking(parser: ConfigParser, settings: dict):
     """
     Patch the loaded bundle config with a new handler logic to enable experiment tracking features.
@@ -497,7 +486,6 @@ def patch_bundle_tracking(parser: ConfigParser, settings: dict):
                 if handlers is None:
                     engine["train_handlers" if k == "trainer" else "val_handlers"] = [handler_config]
                 else:
-                    _add_config_to_mlflow_handler(parser=parser, handler_config=handler_config)
                     handlers.append(handler_config)
 
 

From 1084f0d914e030cb7be318156ba3367254abe9f7 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 23 Nov 2022 08:26:10 +0000
Subject: [PATCH 19/51] fix the type hints in mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/scripts.py          | 1 +
 monai/handlers/mlflow_handler.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index 49576556d1..fc0d2a09b5 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -468,6 +468,7 @@ def get_bundle_info(
 
     return bundle_info[version]
 
+
 def patch_bundle_tracking(parser: ConfigParser, settings: dict):
     """
     Patch the loaded bundle config with a new handler logic to enable experiment tracking features.
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 3db81542e3..16fd29e826 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -106,7 +106,7 @@ def __init__(
         run_name: Optional[str] = None,
         experiment_param: Optional[Dict] = None,
         artifacts: Optional[Sequence[Path]] = None,
-        optimizer_param_names: Tuple[str] = ("lr"),
+        optimizer_param_names: Tuple[str, ...] = ("lr",),
     ) -> None:
         if tracking_uri is not None:
             mlflow.set_tracking_uri(tracking_uri)

From 30e25bce472fc762f92619b431512280778524e3 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 23 Nov 2022 14:23:50 +0000
Subject: [PATCH 20/51] add a new test case for mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 tests/test_handler_mlflow.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py
index 9f8b829481..72ff8db00f 100644
--- a/tests/test_handler_mlflow.py
+++ b/tests/test_handler_mlflow.py
@@ -48,6 +48,13 @@ def _update_metric(engine):
             # check logging output
             self.assertTrue(len(glob.glob(test_path)) > 0)
 
+    def test_input_parameters(self):
+        experiment_param = {"backbone" : "efficientnet_b0"}
+        artifacts_path = tempfile.TemporaryDirectory()
+        handler = MLFlowHandler(experiment_param=experiment_param, artifacts=artifacts_path)
+        self.assertEqual(handler.experiment_param, experiment_param)
+        self.assertEqual(handler.artifacts, artifacts_path)
+
 
 if __name__ == "__main__":
     unittest.main()

From 5d4eb4e9a2ee1a9466742acb79672bcd5f183c5a Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Wed, 23 Nov 2022 14:34:08 +0000
Subject: [PATCH 21/51] fix the code format in mlflow handler test case

Signed-off-by: binliu <binliu@nvidia.com>
---
 tests/test_handler_mlflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py
index 72ff8db00f..0039c150dd 100644
--- a/tests/test_handler_mlflow.py
+++ b/tests/test_handler_mlflow.py
@@ -49,7 +49,7 @@ def _update_metric(engine):
             self.assertTrue(len(glob.glob(test_path)) > 0)
 
     def test_input_parameters(self):
-        experiment_param = {"backbone" : "efficientnet_b0"}
+        experiment_param = {"backbone": "efficientnet_b0"}
         artifacts_path = tempfile.TemporaryDirectory()
         handler = MLFlowHandler(experiment_param=experiment_param, artifacts=artifacts_path)
         self.assertEqual(handler.experiment_param, experiment_param)

From 2dbfc5a8952b7d9eea8888d539169061cc3b67ca Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Sun, 27 Nov 2022 13:41:39 +0000
Subject: [PATCH 22/51] add test case to mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 tests/test_handler_mlflow.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py
index 0039c150dd..725c312746 100644
--- a/tests/test_handler_mlflow.py
+++ b/tests/test_handler_mlflow.py
@@ -13,6 +13,7 @@
 import os
 import tempfile
 import unittest
+import numpy as np
 from pathlib import Path
 
 from ignite.engine import Engine, Events
@@ -22,6 +23,7 @@
 
 class TestHandlerMLFlow(unittest.TestCase):
     def test_metrics_track(self):
+        experiment_param = {"backbone": "efficientnet_b0"}
         with tempfile.TemporaryDirectory() as tempdir:
 
             # set up engine
@@ -39,8 +41,18 @@ def _update_metric(engine):
 
             # set up testing handler
             test_path = os.path.join(tempdir, "mlflow_test")
+            artifact_path = os.path.join(tempdir, "artifacts")
+            os.makedirs(artifact_path,exist_ok=True)
+            dummy_numpy = np.zeros((64, 64, 3))
+            dummy_path =  os.path.join(artifact_path, "tmp.npy")
+            np.save(dummy_path, dummy_numpy)
             handler = MLFlowHandler(
-                iteration_log=False, epoch_log=True, tracking_uri=Path(test_path).as_uri(), state_attributes=["test"]
+                iteration_log=False, \
+                epoch_log=True, \
+                tracking_uri=Path(test_path).as_uri(), \
+                state_attributes=["test"], \
+                experiment_param=experiment_param, \
+                artifacts=[artifact_path]
             )
             handler.attach(engine)
             engine.run(range(3), max_epochs=2)
@@ -48,13 +60,5 @@ def _update_metric(engine):
             # check logging output
             self.assertTrue(len(glob.glob(test_path)) > 0)
 
-    def test_input_parameters(self):
-        experiment_param = {"backbone": "efficientnet_b0"}
-        artifacts_path = tempfile.TemporaryDirectory()
-        handler = MLFlowHandler(experiment_param=experiment_param, artifacts=artifacts_path)
-        self.assertEqual(handler.experiment_param, experiment_param)
-        self.assertEqual(handler.artifacts, artifacts_path)
-
-
 if __name__ == "__main__":
     unittest.main()

From 8ae451a8f60f9eec89395d25852e4023aa4e1522 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Sun, 27 Nov 2022 13:51:44 +0000
Subject: [PATCH 23/51] add a convert in mlflow handler in case the input of
 artifact is a string

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 16fd29e826..61afad4cbb 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -231,6 +231,9 @@ def complete(self) -> None:
         """
         Handler for train or validation/evaluation completed Event.
         """
+        # If just input one path, convert it to list.
+        if isinstance(self.artifacts, str):
+            self.artifacts = [self.artifacts]
         if self.artifacts:
             artifact_list = self._parse_artifacts()
             for artifact in artifact_list:

From 745cb2a8123927058c41ac676977fe348d5c40e4 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Sun, 27 Nov 2022 13:54:08 +0000
Subject: [PATCH 24/51] code format the test file of mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 tests/test_handler_mlflow.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/tests/test_handler_mlflow.py b/tests/test_handler_mlflow.py
index 725c312746..de164cadb8 100644
--- a/tests/test_handler_mlflow.py
+++ b/tests/test_handler_mlflow.py
@@ -13,9 +13,9 @@
 import os
 import tempfile
 import unittest
-import numpy as np
 from pathlib import Path
 
+import numpy as np
 from ignite.engine import Engine, Events
 
 from monai.handlers import MLFlowHandler
@@ -42,17 +42,17 @@ def _update_metric(engine):
             # set up testing handler
             test_path = os.path.join(tempdir, "mlflow_test")
             artifact_path = os.path.join(tempdir, "artifacts")
-            os.makedirs(artifact_path,exist_ok=True)
+            os.makedirs(artifact_path, exist_ok=True)
             dummy_numpy = np.zeros((64, 64, 3))
-            dummy_path =  os.path.join(artifact_path, "tmp.npy")
+            dummy_path = os.path.join(artifact_path, "tmp.npy")
             np.save(dummy_path, dummy_numpy)
             handler = MLFlowHandler(
-                iteration_log=False, \
-                epoch_log=True, \
-                tracking_uri=Path(test_path).as_uri(), \
-                state_attributes=["test"], \
-                experiment_param=experiment_param, \
-                artifacts=[artifact_path]
+                iteration_log=False,
+                epoch_log=True,
+                tracking_uri=Path(test_path).as_uri(),
+                state_attributes=["test"],
+                experiment_param=experiment_param,
+                artifacts=[artifact_path],
             )
             handler.attach(engine)
             engine.run(range(3), max_epochs=2)
@@ -60,5 +60,6 @@ def _update_metric(engine):
             # check logging output
             self.assertTrue(len(glob.glob(test_path)) > 0)
 
+
 if __name__ == "__main__":
     unittest.main()

From a73de6f62b5c87ab1758a56d9699dd1037826d57 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 06:46:43 +0000
Subject: [PATCH 25/51] add DEFAULT_EXP_MGMT_SETTINGS to bundle init.

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/__init__.py | 2 +-
 monai/bundle/utils.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py
index fba3aee994..28b9c78073 100644
--- a/monai/bundle/__init__.py
+++ b/monai/bundle/__init__.py
@@ -25,4 +25,4 @@
     verify_metadata,
     verify_net_in_out,
 )
-from .utils import DEFAULT_EXP_MGMT_SETTINGS, EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config
+from .utils import DEFAULT_EXP_MGMT_SETTINGS, DEFAULT_MLFLOW_SETTINGS, EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config
diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 4f5b9615bc..14d54084b6 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -19,7 +19,7 @@
 
 yaml, _ = optional_import("yaml")
 
-__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS"]
+__all__ = ["ID_REF_KEY", "ID_SEP_KEY", "EXPR_KEY", "MACRO_KEY", "DEFAULT_MLFLOW_SETTINGS", "DEFAULT_EXP_MGMT_SETTINGS"]
 
 ID_REF_KEY = "@"  # start of a reference to a ConfigItem
 ID_SEP_KEY = "#"  # separator for the ID of a ConfigItem

From d1d663a245954bda87ed25feec5902431adc06f4 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 06:48:07 +0000
Subject: [PATCH 26/51] fix code format for bundle utils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/__init__.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/monai/bundle/__init__.py b/monai/bundle/__init__.py
index 28b9c78073..c5cb0da978 100644
--- a/monai/bundle/__init__.py
+++ b/monai/bundle/__init__.py
@@ -25,4 +25,12 @@
     verify_metadata,
     verify_net_in_out,
 )
-from .utils import DEFAULT_EXP_MGMT_SETTINGS, DEFAULT_MLFLOW_SETTINGS, EXPR_KEY, ID_REF_KEY, ID_SEP_KEY, MACRO_KEY, load_bundle_config
+from .utils import (
+    DEFAULT_EXP_MGMT_SETTINGS,
+    DEFAULT_MLFLOW_SETTINGS,
+    EXPR_KEY,
+    ID_REF_KEY,
+    ID_SEP_KEY,
+    MACRO_KEY,
+    load_bundle_config,
+)

From 653d4af67c017fef1ba41b08caf6470e9b3e12e8 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 06:54:37 +0000
Subject: [PATCH 27/51] add is_avaliable check for initial configs in mlflow
 handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 14d54084b6..812eb1b1b2 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -108,7 +108,7 @@
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
             "iteration_log": True,
             "epoch_log": True,
@@ -118,14 +118,14 @@
         # MLFlowHandler config for the validator
         "validator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
             "iteration_log": False,
         },
         # MLFlowHandler config for the evaluator
         "evaluator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if torch.distributed.is_initialized() else False",
+            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
             "iteration_log": False,
         },

From a8be4de5905b12569804dc6cc7bd2a9518d4b7f3 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 07:20:42 +0000
Subject: [PATCH 28/51] delete seed from the default attribute

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 61afad4cbb..b91fbec389 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -89,7 +89,7 @@ class MLFlowHandler:
     """
 
     default_log_param_list = ["network", "device", "optimizer", "loss_function"]
-    default_attr_name = ["seed", "max_epochs", "epoch_length"]
+    default_attr_name = ["max_epochs", "epoch_length"]
 
     def __init__(
         self,

From 70f13964030fe9cd002980ad300c509e42c718c1 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 08:20:52 +0000
Subject: [PATCH 29/51] delete workflow param record, convert artifacts at init
 and change the try exception logic

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py            |  8 ++++++++
 monai/handlers/mlflow_handler.py | 17 ++++-------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 812eb1b1b2..7d7663fe3b 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -105,11 +105,15 @@
     "handlers_id": DEFAULT_HANDLERS_ID,
     "configs": {
         "tracking_uri": "$@output_dir + '/mlruns'",
+        "experiment_name" : "default_experiment",
+        "run_name": "$None",
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
             "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
+            "experiment_name": "@experiment_name",
+            "run_name": "@run_name",
             "iteration_log": True,
             "epoch_log": True,
             "tag_name": "train_loss",
@@ -120,6 +124,8 @@
             "_target_": "MLFlowHandler",
             "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
+            "experiment_name": "@experiment_name",
+            "run_name": "@run_name",
             "iteration_log": False,
         },
         # MLFlowHandler config for the evaluator
@@ -127,6 +133,8 @@
             "_target_": "MLFlowHandler",
             "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
             "tracking_uri": "@tracking_uri",
+            "experiment_name": "@experiment_name",
+            "run_name": "@run_name",
             "iteration_log": False,
         },
     },
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index b91fbec389..c6ac932449 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -18,7 +18,7 @@
 
 from monai.config import IgniteInfo
 from monai.engines import Trainer
-from monai.utils import min_version, optional_import
+from monai.utils import min_version, optional_import, ensure_tuple
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
 mlflow, _ = optional_import("mlflow")
@@ -88,7 +88,6 @@ class MLFlowHandler:
 
     """
 
-    default_log_param_list = ["network", "device", "optimizer", "loss_function"]
     default_attr_name = ["max_epochs", "epoch_length"]
 
     def __init__(
@@ -122,7 +121,7 @@ def __init__(
         self.experiment_name = experiment_name
         self.run_name = run_name
         self.experiment_param = experiment_param
-        self.artifacts = artifacts
+        self.artifacts = ensure_tuple(artifacts) if artifacts else None
         self.optimizer_param_names = optimizer_param_names
         self.client = mlflow.MlflowClient()
 
@@ -197,7 +196,7 @@ def start(self, engine: Engine) -> None:
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:
-            # avoid to recording same parameters in the same run
+            # avoid to record same parameters in the same run
             self._delete_exist_param_in_dict(self.experiment_param)
             if self.experiment_param:
                 mlflow.log_params(self.experiment_param)
@@ -206,11 +205,6 @@ def start(self, engine: Engine) -> None:
         self._delete_exist_param_in_dict(attrs)
         mlflow.log_params(attrs)
 
-        for param_name in self.default_log_param_list:
-            if self._is_param_exists(param_name):
-                continue
-            self._try_log_param(engine, param_name)
-
     def _parse_artifacts(self):
         """
         Log artifacts to mlflow. Given a path, all files in the path will be logged recursively.
@@ -317,7 +311,7 @@ def _default_iteration_log(self, engine: Engine) -> None:
             mlflow.log_metrics(loss, step=engine.state.iteration)
 
         # If there is optimizer attr in engine, then record parameters specified in init function.
-        try:
+        if hasattr(engine, 'optimizer'):
             cur_optimizer = engine.optimizer  # type: ignore
             for param_name in self.optimizer_param_names:
                 params = {
@@ -325,6 +319,3 @@ def _default_iteration_log(self, engine: Engine) -> None:
                     for i, param_group in enumerate(cur_optimizer.param_groups)
                 }
                 mlflow.log_metrics(params, step=engine.state.iteration)
-
-        except AttributeError:
-            pass

From ac063e30b9e04dd80e2def43929d6c1fe2f2f235 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 08:21:40 +0000
Subject: [PATCH 30/51] delete the list convert in complete function

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index c6ac932449..67fad4af22 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -225,9 +225,6 @@ def complete(self) -> None:
         """
         Handler for train or validation/evaluation completed Event.
         """
-        # If just input one path, convert it to list.
-        if isinstance(self.artifacts, str):
-            self.artifacts = [self.artifacts]
         if self.artifacts:
             artifact_list = self._parse_artifacts()
             for artifact in artifact_list:

From 958521bfc8abff3b481199dabbdf57f1b025ec96 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 08:24:13 +0000
Subject: [PATCH 31/51] fix code format for mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py            | 2 +-
 monai/handlers/mlflow_handler.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 7d7663fe3b..922bacf172 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -105,7 +105,7 @@
     "handlers_id": DEFAULT_HANDLERS_ID,
     "configs": {
         "tracking_uri": "$@output_dir + '/mlruns'",
-        "experiment_name" : "default_experiment",
+        "experiment_name": "default_experiment",
         "run_name": "$None",
         # MLFlowHandler config for the trainer
         "trainer": {
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 67fad4af22..cd8147e516 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -18,7 +18,7 @@
 
 from monai.config import IgniteInfo
 from monai.engines import Trainer
-from monai.utils import min_version, optional_import, ensure_tuple
+from monai.utils import ensure_tuple, min_version, optional_import
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")
 mlflow, _ = optional_import("mlflow")
@@ -308,7 +308,7 @@ def _default_iteration_log(self, engine: Engine) -> None:
             mlflow.log_metrics(loss, step=engine.state.iteration)
 
         # If there is optimizer attr in engine, then record parameters specified in init function.
-        if hasattr(engine, 'optimizer'):
+        if hasattr(engine, "optimizer"):
             cur_optimizer = engine.optimizer  # type: ignore
             for param_name in self.optimizer_param_names:
                 params = {

From 6bacc1604fbbf93b2fc9a31d474ec51efa50af24 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 12:17:45 +0000
Subject: [PATCH 32/51] update artifacts saving code for mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index cd8147e516..2317b7ee35 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -121,7 +121,7 @@ def __init__(
         self.experiment_name = experiment_name
         self.run_name = run_name
         self.experiment_param = experiment_param
-        self.artifacts = ensure_tuple(artifacts) if artifacts else None
+        self.artifacts = ensure_tuple(artifacts)
         self.optimizer_param_names = optimizer_param_names
         self.client = mlflow.MlflowClient()
 
@@ -212,6 +212,9 @@ def _parse_artifacts(self):
         """
         artifact_list = []
         for path_name in self.artifacts:
+            # in case the input is (None,) by default
+            if not path_name:
+                continue
             if os.path.isfile(path_name):
                 artifact_list.append(path_name)
             else:

From 181ad9197e31ec24a2f5d5909afaf9435a7d5078 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 12:19:45 +0000
Subject: [PATCH 33/51] remove the code for deleting repeat parameters in
 experiment parameters

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 2317b7ee35..43d183f867 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -196,10 +196,7 @@ def start(self, engine: Engine) -> None:
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:
-            # avoid to record same parameters in the same run
-            self._delete_exist_param_in_dict(self.experiment_param)
-            if self.experiment_param:
-                mlflow.log_params(self.experiment_param)
+            mlflow.log_params(self.experiment_param)
 
         attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
         self._delete_exist_param_in_dict(attrs)

From 73078fe9bc2a7481f33d89de9ffd3d931a70befd Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 12:56:30 +0000
Subject: [PATCH 34/51] fix the long string format problem in bundle/utils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 922bacf172..62afac7180 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -110,7 +110,11 @@
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
+            "_disabled_": (
+                "$torch.distributed.get_rank() > 0 \
+                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
+                else False"
+            ),
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -122,7 +126,11 @@
         # MLFlowHandler config for the validator
         "validator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
+            "_disabled_": (
+                "$torch.distributed.get_rank() > 0 \
+                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
+                    else False"
+            ),
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -131,7 +139,11 @@
         # MLFlowHandler config for the evaluator
         "evaluator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "$torch.distributed.get_rank() > 0 if (torch.distributed.is_available() and torch.distributed.is_initialized()) else False",
+            "_disabled_": (
+                "$torch.distributed.get_rank() > 0 \
+                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
+                else False"
+            ),
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",

From c8d0f392c80e031e91f6aa806e7e4eb2fbe30c7a Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Mon, 28 Nov 2022 14:55:44 +0000
Subject: [PATCH 35/51] change the rtol value in
 test_scale_intensity_range_percentiles since it won't pass the ci/cd test in
 this version

Signed-off-by: binliu <binliu@nvidia.com>
---
 tests/test_scale_intensity_range_percentiles.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_scale_intensity_range_percentiles.py b/tests/test_scale_intensity_range_percentiles.py
index 184e1dff0c..181243b0fe 100644
--- a/tests/test_scale_intensity_range_percentiles.py
+++ b/tests/test_scale_intensity_range_percentiles.py
@@ -58,7 +58,7 @@ def test_relative_scaling(self):
         for p in TEST_NDARRAYS:
             result = scaler(p(img))
             assert_allclose(
-                result, p(np.clip(expected_img, expected_b_min, expected_b_max)), type_test="tensor", rtol=1e-4
+                result, p(np.clip(expected_img, expected_b_min, expected_b_max)), type_test="tensor", rtol=0.1
             )
 
     def test_invalid_instantiation(self):

From 91b925bf4fabf27b0a36a9e4dad0335d51801d9f Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 04:44:20 +0000
Subject: [PATCH 36/51] formate some code type

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 62afac7180..7a0ffb5733 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -107,14 +107,15 @@
         "tracking_uri": "$@output_dir + '/mlruns'",
         "experiment_name": "default_experiment",
         "run_name": "$None",
-        # MLFlowHandler config for the trainer
-        "trainer": {
-            "_target_": "MLFlowHandler",
-            "_disabled_": (
+        "not_rank0": (
                 "$torch.distributed.get_rank() > 0 \
                 if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
                 else False"
             ),
+        # MLFlowHandler config for the trainer
+        "trainer": {
+            "_target_": "MLFlowHandler",
+            "_disabled_": "@not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -126,11 +127,7 @@
         # MLFlowHandler config for the validator
         "validator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": (
-                "$torch.distributed.get_rank() > 0 \
-                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
-                    else False"
-            ),
+            "_disabled_": "@not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -139,11 +136,7 @@
         # MLFlowHandler config for the evaluator
         "evaluator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": (
-                "$torch.distributed.get_rank() > 0 \
-                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
-                else False"
-            ),
+            "_disabled_": "@not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",

From cabc6cc56e78cf54c640ec3e22efa35eb6dc7da1 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 04:46:11 +0000
Subject: [PATCH 37/51] fix code format in bundle/utils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 7a0ffb5733..d4f49e15e1 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -108,10 +108,10 @@
         "experiment_name": "default_experiment",
         "run_name": "$None",
         "not_rank0": (
-                "$torch.distributed.get_rank() > 0 \
+            "$torch.distributed.get_rank() > 0 \
                 if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
                 else False"
-            ),
+        ),
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",

From 28aae2008d753d507b0b6d92b4bdbbd66a5e3854 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 07:35:53 +0000
Subject: [PATCH 38/51] change the default value of mlflow run name in untils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index d4f49e15e1..e11e1ea47c 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -106,7 +106,7 @@
     "configs": {
         "tracking_uri": "$@output_dir + '/mlruns'",
         "experiment_name": "default_experiment",
-        "run_name": "$None",
+        "run_name": "None",
         "not_rank0": (
             "$torch.distributed.get_rank() > 0 \
                 if (torch.distributed.is_available() and torch.distributed.is_initialized()) \

From c564df46202515f7baf37333a8f84d5275f0a966 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 07:37:41 +0000
Subject: [PATCH 39/51] change the not_rank0 name to is_not_rank0 in
 bundle/utils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index e11e1ea47c..15dad5831e 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -107,7 +107,7 @@
         "tracking_uri": "$@output_dir + '/mlruns'",
         "experiment_name": "default_experiment",
         "run_name": "None",
-        "not_rank0": (
+        "is_not_rank0": (
             "$torch.distributed.get_rank() > 0 \
                 if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
                 else False"
@@ -115,7 +115,7 @@
         # MLFlowHandler config for the trainer
         "trainer": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "@not_rank0",
+            "_disabled_": "@is_not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -127,7 +127,7 @@
         # MLFlowHandler config for the validator
         "validator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "@not_rank0",
+            "_disabled_": "@is_not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",
@@ -136,7 +136,7 @@
         # MLFlowHandler config for the evaluator
         "evaluator": {
             "_target_": "MLFlowHandler",
-            "_disabled_": "@not_rank0",
+            "_disabled_": "@is_not_rank0",
             "tracking_uri": "@tracking_uri",
             "experiment_name": "@experiment_name",
             "run_name": "@run_name",

From 1e95f12621b3b3e34b24073dd2c021c57d8afe62 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 07:47:45 +0000
Subject: [PATCH 40/51] change some arguments' type in mlflow hander

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 43d183f867..64b907f1f4 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -12,7 +12,7 @@
 import os
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Tuple
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Tuple, Union
 
 import torch
 
@@ -104,8 +104,8 @@ def __init__(
         experiment_name: str = "default_experiment",
         run_name: Optional[str] = None,
         experiment_param: Optional[Dict] = None,
-        artifacts: Optional[Sequence[Path]] = None,
-        optimizer_param_names: Tuple[str, ...] = ("lr",),
+        artifacts: Optional[Union[str, Sequence[Path]]] = None,
+        optimizer_param_names: Union[str, Sequence[str]] = "lr",
     ) -> None:
         if tracking_uri is not None:
             mlflow.set_tracking_uri(tracking_uri)
@@ -122,7 +122,7 @@ def __init__(
         self.run_name = run_name
         self.experiment_param = experiment_param
         self.artifacts = ensure_tuple(artifacts)
-        self.optimizer_param_names = optimizer_param_names
+        self.optimizer_param_names = ensure_tuple(optimizer_param_names)
         self.client = mlflow.MlflowClient()
 
     def _try_log_param(self, engine: Engine, attr: str) -> None:

From b7fcff139d17d0ee2ae437d11b5389f8f18eaac0 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 07:49:00 +0000
Subject: [PATCH 41/51] delete a legacy method in mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 64b907f1f4..ad61c4aa44 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -125,19 +125,6 @@ def __init__(
         self.optimizer_param_names = ensure_tuple(optimizer_param_names)
         self.client = mlflow.MlflowClient()
 
-    def _try_log_param(self, engine: Engine, attr: str) -> None:
-        """
-        Log parameter to mlflow if it exists in given engine.
-
-        Args:
-            engine: A ignite engine.
-            attr: Attribute that needs to be logged by mlflow.
-        """
-        engine_attr = getattr(engine, attr, None)
-        if engine_attr:
-            attr_type_string = str(type(engine_attr))
-            mlflow.log_param(key=attr, value=attr_type_string)
-
     def _is_param_exists(self, param_name: str) -> bool:
         """
         Check whether a parameter has already been logged in current mlflow run.

From 1c7437b48ba4cf6b805197076989efa78f3a016e Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 08:06:07 +0000
Subject: [PATCH 42/51] remove the trainer check logic from the mlflow handler
 since users should set iteration_log=False to avoid this

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index ad61c4aa44..25d5226860 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -12,7 +12,7 @@
 import os
 import time
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence, Union
 
 import torch
 
@@ -291,8 +291,7 @@ def _default_iteration_log(self, engine: Engine) -> None:
         if not isinstance(loss, dict):
             loss = {self.tag_name: loss.item() if isinstance(loss, torch.Tensor) else loss}
 
-        if isinstance(engine, Trainer):
-            mlflow.log_metrics(loss, step=engine.state.iteration)
+        mlflow.log_metrics(loss, step=engine.state.iteration)
 
         # If there is optimizer attr in engine, then record parameters specified in init function.
         if hasattr(engine, "optimizer"):

From 608ce24c8ac570fe158078c90a501f20e0d40cab Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 08:09:30 +0000
Subject: [PATCH 43/51] remove the Trainer import, since it won't be used in
 mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 25d5226860..6b00722eef 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -17,7 +17,6 @@
 import torch
 
 from monai.config import IgniteInfo
-from monai.engines import Trainer
 from monai.utils import ensure_tuple, min_version, optional_import
 
 Events, _ = optional_import("ignite.engine", IgniteInfo.OPT_IMPORT_VERSION, min_version, "Events")

From ecab937d5b269380b5f51fc6f2f3810b5be1e0cf Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 12:44:56 +0000
Subject: [PATCH 44/51] change default name to monai experiment and change the
 rank0 logic in bundle/utils

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 15dad5831e..0c760d69fb 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -105,12 +105,11 @@
     "handlers_id": DEFAULT_HANDLERS_ID,
     "configs": {
         "tracking_uri": "$@output_dir + '/mlruns'",
-        "experiment_name": "default_experiment",
+        "experiment_name": "monai_experiment",
         "run_name": "None",
         "is_not_rank0": (
-            "$torch.distributed.get_rank() > 0 \
-                if (torch.distributed.is_available() and torch.distributed.is_initialized()) \
-                else False"
+            "$torch.distributed.is_available() \
+                and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0"
         ),
         # MLFlowHandler config for the trainer
         "trainer": {

From cb7ae68ef7642361388850e769c8571c036f8a80 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 13:05:36 +0000
Subject: [PATCH 45/51] merge two functions into one

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 6b00722eef..566ec50731 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -124,24 +124,6 @@ def __init__(
         self.optimizer_param_names = ensure_tuple(optimizer_param_names)
         self.client = mlflow.MlflowClient()
 
-    def _is_param_exists(self, param_name: str) -> bool:
-        """
-        Check whether a parameter has already been logged in current mlflow run.
-
-        Args:
-            param_name: name of parameter to check in mlflow run.
-
-        Return:
-            If the given parameter already was logged to mlflow.
-        """
-        cur_run = mlflow.active_run()
-        log_data = self.client.get_run(cur_run.info.run_id).data
-        param_dict = log_data.params
-        if param_name in param_dict:
-            return True
-        else:
-            return False
-
     def _delete_exist_param_in_dict(self, param_dict: Dict) -> None:
         """
         Delete parameters in given dict, if they are already logged by current mlflow run.
@@ -150,8 +132,11 @@ def _delete_exist_param_in_dict(self, param_dict: Dict) -> None:
             param_dict: parameter dict to be logged to mlflow.
         """
         key_list = list(param_dict.keys())
+        cur_run = mlflow.active_run()
+        log_data = self.client.get_run(cur_run.info.run_id).data
+        log_param_dict = log_data.params
         for key in key_list:
-            if self._is_param_exists(key):
+            if key in log_param_dict:
                 del param_dict[key]
 
     def attach(self, engine: Engine) -> None:
@@ -178,7 +163,8 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            run_name = f"run_{time.strftime('%Y%m%d_%H%M%S')}" if self.run_name is None else self.run_name
+            run_name = (f"run_{time.strftime('%Y%m%d_%H%M%S')}" \
+                if (self.run_name is None or self.run_name == "None") else self.run_name)
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:

From 48507f3246a8039439ad1ee546e667d6ea15d5f2 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Thu, 1 Dec 2022 13:08:02 +0000
Subject: [PATCH 46/51] format code style in mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 566ec50731..eb9a8bc84c 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -163,8 +163,11 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            run_name = (f"run_{time.strftime('%Y%m%d_%H%M%S')}" \
-                if (self.run_name is None or self.run_name == "None") else self.run_name)
+            run_name = (
+                f"run_{time.strftime('%Y%m%d_%H%M%S')}"
+                if (self.run_name is None or self.run_name == "None")
+                else self.run_name
+            )
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:

From a8fef29109285ba797c8df8c8423f59d6232339c Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 2 Dec 2022 02:23:37 +0000
Subject: [PATCH 47/51] change default run name in mlflow to None

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/utils.py            | 2 +-
 monai/handlers/mlflow_handler.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/monai/bundle/utils.py b/monai/bundle/utils.py
index 0c760d69fb..4b1adeffea 100644
--- a/monai/bundle/utils.py
+++ b/monai/bundle/utils.py
@@ -106,7 +106,7 @@
     "configs": {
         "tracking_uri": "$@output_dir + '/mlruns'",
         "experiment_name": "monai_experiment",
-        "run_name": "None",
+        "run_name": None,
         "is_not_rank0": (
             "$torch.distributed.is_available() \
                 and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0"
diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index eb9a8bc84c..3ca9da08b7 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -165,7 +165,7 @@ def start(self, engine: Engine) -> None:
         if mlflow.active_run() is None:
             run_name = (
                 f"run_{time.strftime('%Y%m%d_%H%M%S')}"
-                if (self.run_name is None or self.run_name == "None")
+                if (self.run_name is None)
                 else self.run_name
             )
             mlflow.start_run(run_name=run_name)

From de48bee4547c702cc08c191073a30713c95cbbb0 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 2 Dec 2022 02:25:13 +0000
Subject: [PATCH 48/51] update doc string of default value of
 optimizer_param_names

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 3ca9da08b7..3f7c2ddf4c 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -81,7 +81,7 @@ class MLFlowHandler:
             like torch version, cuda version and so on.
         artifacts: paths to images that need to be recorded after a whole run.
         optimizer_param_names: parameters' name in optimizer that need to be record during runing,
-            defaults to `["lr"]`.
+            defaults to "lr".
 
     For more details of MLFlow usage, please refer to: https://mlflow.org/docs/latest/index.html.
 

From 56fbb0531fd0722af6bc7fd3decd6e39fc3c97f5 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 2 Dec 2022 02:36:38 +0000
Subject: [PATCH 49/51] udpate default mlflow setting docstring in
 bundle/script

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/bundle/scripts.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/monai/bundle/scripts.py b/monai/bundle/scripts.py
index ace76ac75c..0ce5a9b607 100644
--- a/monai/bundle/scripts.py
+++ b/monai/bundle/scripts.py
@@ -547,17 +547,36 @@ def run(
                     },
                     "configs": {
                         "tracking_uri": "<path>",
+                        "experiment_name": "monai_experiment",
+                        "run_name": None,
+                        "is_not_rank0": (
+                            "$torch.distributed.is_available() \
+                                and torch.distributed.is_initialized() and torch.distributed.get_rank() > 0"
+                        ),
                         "trainer": {
                             "_target_": "MLFlowHandler",
+                            "_disabled_": "@is_not_rank0",
                             "tracking_uri": "@tracking_uri",
+                            "experiment_name": "@experiment_name",
+                            "run_name": "@run_name",
                             "iteration_log": True,
                             "output_transform": "$monai.handlers.from_engine(['loss'], first=True)",
                         },
                         "validator": {
-                            "_target_": "MLFlowHandler", "tracking_uri": "@tracking_uri", "iteration_log": False,
+                            "_target_": "MLFlowHandler",
+                            "_disabled_": "@is_not_rank0",
+                            "tracking_uri": "@tracking_uri",
+                            "experiment_name": "@experiment_name",
+                            "run_name": "@run_name",
+                            "iteration_log": False,
                         },
                         "evaluator": {
-                            "_target_": "MLFlowHandler", "tracking_uri": "@tracking_uri", "iteration_log": False,
+                            "_target_": "MLFlowHandler",
+                            "_disabled_": "@is_not_rank0",
+                            "tracking_uri": "@tracking_uri",
+                            "experiment_name": "@experiment_name",
+                            "run_name": "@run_name",
+                            "iteration_log": False,
                         },
                     },
                 },

From 850f98f8575dd70d79d158a7012392af29cafb05 Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 2 Dec 2022 02:42:52 +0000
Subject: [PATCH 50/51] fix code format issue in mlflow handler

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 3f7c2ddf4c..698f74963a 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -163,11 +163,7 @@ def start(self, engine: Engine) -> None:
         """
         mlflow.set_experiment(self.experiment_name)
         if mlflow.active_run() is None:
-            run_name = (
-                f"run_{time.strftime('%Y%m%d_%H%M%S')}"
-                if (self.run_name is None)
-                else self.run_name
-            )
+            run_name = f"run_{time.strftime('%Y%m%d_%H%M%S')}" if self.run_name is None else self.run_name
             mlflow.start_run(run_name=run_name)
 
         if self.experiment_param:

From ebb804cecb236d0a10b41a84c1ed9954394af68c Mon Sep 17 00:00:00 2001
From: binliu <binliu@nvidia.com>
Date: Fri, 2 Dec 2022 15:23:11 +0000
Subject: [PATCH 51/51] change the default_attr_name to default_tracking_params
 to avoid confusion

Signed-off-by: binliu <binliu@nvidia.com>
---
 monai/handlers/mlflow_handler.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/monai/handlers/mlflow_handler.py b/monai/handlers/mlflow_handler.py
index 698f74963a..277903e8e2 100644
--- a/monai/handlers/mlflow_handler.py
+++ b/monai/handlers/mlflow_handler.py
@@ -87,7 +87,8 @@ class MLFlowHandler:
 
     """
 
-    default_attr_name = ["max_epochs", "epoch_length"]
+    # parameters that are logged at the start of training
+    default_tracking_params = ["max_epochs", "epoch_length"]
 
     def __init__(
         self,
@@ -169,7 +170,7 @@ def start(self, engine: Engine) -> None:
         if self.experiment_param:
             mlflow.log_params(self.experiment_param)
 
-        attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_attr_name}
+        attrs = {attr: getattr(engine.state, attr, None) for attr in self.default_tracking_params}
         self._delete_exist_param_in_dict(attrs)
         mlflow.log_params(attrs)