From 2f33272bb44c6b3201116d27d08164eb8b6eee2e Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Tue, 23 Aug 2022 16:29:27 -0400 Subject: [PATCH 1/8] warn if no weights were loaded Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index 175ed05d67..c88b76338d 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -233,7 +233,9 @@ def train(self, data: ExchangeObject, extra=None): # get current iteration when a round starts self.iter_of_start_time = self.trainer.state.iteration - copy_model_state(src=self.global_weights, dst=self.trainer.network) + _, updated_keys, _ = copy_model_state(src=self.global_weights, dst=self.trainer.network) + if len(updated_keys) == 0: + self.logger.warning("No weights loaded!") self.logger.info(f"Start {self.client_name} training...") self.trainer.run() @@ -311,7 +313,9 @@ def evaluate(self, data: ExchangeObject, extra=None): global_weights=data.weights, local_var_dict=get_state_dict(self.evaluator.network) ) - copy_model_state(src=global_weights, dst=self.evaluator.network) + _, updated_keys, _ = copy_model_state(src=global_weights, dst=self.evaluator.network) + if len(updated_keys) == 0: + self.logger.warning("No weights loaded!") self.logger.info(f"Start {self.client_name} evaluating...") if isinstance(self.trainer, monai.engines.Trainer): self.evaluator.run(self.trainer.state.epoch + 1) From 64f889d643ed1959824b69683c9f5d56b1c1dc88 Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Tue, 23 Aug 2022 17:24:08 -0400 Subject: [PATCH 2/8] disable checkpoint loaders Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 22 +++++++++++++++++++--- monai/fl/utils/constants.py | 1 + 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index c88b76338d..4d3aec6bd2 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -81,6 +81,19 @@ def check_bundle_config(parser): raise KeyError(f"Bundle config misses required key `{k}`") +def remove_ckpt_loader(parser): + if BundleKeys.VALIDATE_HANDLERS in parser: + _handlers = parser.get(BundleKeys.VALIDATE_HANDLERS) + _filtered_handlers = [] + for _h in _handlers: + if isinstance(_h, dict): + if "_target_" in _h: + if "CheckpointLoader" in _h.get("_target_"): + continue + _filtered_handlers.append(_h) + parser[BundleKeys.VALIDATE_HANDLERS] = _filtered_handlers + + class MonaiAlgo(ClientAlgo): """ Implementation of ``ClientAlgo`` to allow federated learning with MONAI bundle configurations. @@ -102,6 +115,7 @@ def __init__( config_train_filename: Optional[str] = "configs/train.json", config_evaluate_filename: Optional[str] = "configs/evaluate.json", config_filters_filename: Optional[str] = None, + disable_ckpt_loader: bool = True, ): self.logger = logging.getLogger(self.__class__.__name__) @@ -111,6 +125,7 @@ def __init__( self.config_train_filename = config_train_filename self.config_evaluate_filename = config_evaluate_filename self.config_filters_filename = config_filters_filename + self.disable_ckpt_loader = disable_ckpt_loader self.app_root = None self.train_parser = None @@ -178,9 +193,10 @@ def initialize(self, extra=None): if BundleKeys.TRAIN_TRAINER_MAX_EPOCHS in self.train_parser: self.train_parser[BundleKeys.TRAIN_TRAINER_MAX_EPOCHS] = self.local_epochs - self.train_parser.parse() - self.eval_parser.parse() - self.filter_parser.parse() + # remove checkpoint loaders + if self.disable_ckpt_loader: + remove_ckpt_loader(self.train_parser) + remove_ckpt_loader(self.eval_parser) # Get trainer, evaluator self.trainer = self.train_parser.get_parsed_content( diff --git a/monai/fl/utils/constants.py b/monai/fl/utils/constants.py index 90b452e70d..2d3a8b28bc 100644 --- a/monai/fl/utils/constants.py +++ b/monai/fl/utils/constants.py @@ -43,6 +43,7 @@ class BundleKeys(StrEnum): TRAINER = "train#trainer" EVALUATOR = "validate#evaluator" TRAIN_TRAINER_MAX_EPOCHS = "train#trainer#max_epochs" + VALIDATE_HANDLERS = "validate#handlers" class FiltersType(StrEnum): From 34ea436bd4c814fca8cf14472f489dd7a1cda37d Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Tue, 23 Aug 2022 17:33:17 -0400 Subject: [PATCH 3/8] update doc string Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index 4d3aec6bd2..82280a137f 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -102,9 +102,10 @@ class MonaiAlgo(ClientAlgo): bundle_root: path of bundle. local_epochs: number of local epochs to execute during each round of local training; defaults to 1. send_weight_diff: whether to send weight differences rather than full weights; defaults to `True`. - config_train_filename: bundle training config path relative to bundle_root; defaults to "configs/train.json" - config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json" + config_train_filename: bundle training config path relative to bundle_root; defaults to "configs/train.json". + config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json". config_filters_filename: filter configuration file. + disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to True. """ def __init__( From 1017ae0a5ff3a67b58e79b443ba783b80f72f7cf Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Tue, 23 Aug 2022 20:10:55 -0400 Subject: [PATCH 4/8] support sending best weights Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 49 +++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index 82280a137f..ce507760e3 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -106,6 +106,7 @@ class MonaiAlgo(ClientAlgo): config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json". config_filters_filename: filter configuration file. disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to True. + best_model_filepath: location of best model checkpoint (defaults "model/model.pt" relative to `bundle_root`) """ def __init__( @@ -117,6 +118,7 @@ def __init__( config_evaluate_filename: Optional[str] = "configs/evaluate.json", config_filters_filename: Optional[str] = None, disable_ckpt_loader: bool = True, + best_model_filepath: Optional[str] = "models/model.pt", ): self.logger = logging.getLogger(self.__class__.__name__) @@ -127,6 +129,7 @@ def __init__( self.config_evaluate_filename = config_evaluate_filename self.config_filters_filename = config_filters_filename self.disable_ckpt_loader = disable_ckpt_loader + self.best_model_filepath = best_model_filepath self.app_root = None self.train_parser = None @@ -264,26 +267,46 @@ def get_weights(self, extra=None): extra: Dict with additional information that can be provided by FL system. Returns: - return_weights: `ExchangeObject` containing current weights. + return_weights: `ExchangeObject` containing current weights (default) + or "best" local weights if `extra` has `ExtraItems.MODEL_NAME` with string containing "best". """ if extra is None: extra = {} + + # by default return current weights, return best if requested via model name. + return_best = False + if ExtraItems.MODEL_NAME in extra: + if "best" in extra.get(ExtraItems.MODEL_NAME).lower(): + return_best = True self.phase = FlPhase.GET_WEIGHTS - if self.trainer: - weights = get_state_dict(self.trainer.network) + + if return_best: + self.best_model_filepath = os.path.join(self.bundle_root, self.best_model_filepath) + if not os.path.isfile(self.best_model_filepath): + raise ValueError(f"No best model checkpoint exists at {self.best_model_filepath}") + weights = torch.load(self.best_model_filepath, map_location="cpu") weigh_type = WeightType.WEIGHTS - stats = self.trainer.get_stats() - # calculate current iteration and epoch data after training. - stats[FlStatistics.NUM_EXECUTED_ITERATIONS] = self.trainer.state.iteration - self.iter_of_start_time - # compute weight differences - if self.send_weight_diff: - weights = compute_weight_diff(global_weights=self.global_weights, local_var_dict=weights) - weigh_type = WeightType.WEIGHT_DIFF - else: - weights = None - weigh_type = None stats = dict() + self.logger.info(f"Returning best checkpoint weights from {self.best_model_filepath}.") + else: + if self.trainer: + weights = get_state_dict(self.trainer.network) + weigh_type = WeightType.WEIGHTS + stats = self.trainer.get_stats() + # calculate current iteration and epoch data after training. + stats[FlStatistics.NUM_EXECUTED_ITERATIONS] = self.trainer.state.iteration - self.iter_of_start_time + # compute weight differences + if self.send_weight_diff: + weights = compute_weight_diff(global_weights=self.global_weights, local_var_dict=weights) + weigh_type = WeightType.WEIGHT_DIFF + self.logger.info("Returning current weight differences.") + else: + self.logger.info("Returning current weights.") + else: + weights = None + weigh_type = None + stats = dict() if not isinstance(stats, dict): raise ValueError(f"stats is not a dict, {stats}") From 336699ea00b65d39eb68e4c4ac340aded2d4f8bd Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Wed, 24 Aug 2022 12:16:59 -0400 Subject: [PATCH 5/8] only disable ckpt loader Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index ce507760e3..942c623a40 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -18,7 +18,7 @@ import monai from monai.bundle import ConfigParser -from monai.bundle.config_item import ConfigItem +from monai.bundle.config_item import ConfigItem, ConfigComponent from monai.config import IgniteInfo from monai.fl.client.client_algo import ClientAlgo from monai.fl.utils.constants import ( @@ -83,15 +83,10 @@ def check_bundle_config(parser): def remove_ckpt_loader(parser): if BundleKeys.VALIDATE_HANDLERS in parser: - _handlers = parser.get(BundleKeys.VALIDATE_HANDLERS) - _filtered_handlers = [] - for _h in _handlers: - if isinstance(_h, dict): - if "_target_" in _h: - if "CheckpointLoader" in _h.get("_target_"): - continue - _filtered_handlers.append(_h) - parser[BundleKeys.VALIDATE_HANDLERS] = _filtered_handlers + for h in parser[BundleKeys.VALIDATE_HANDLERS]: + if ConfigComponent.is_instantiable(h): + if "CheckpointLoader" in h["_target_"]: + h["_disabled_"] = True class MonaiAlgo(ClientAlgo): From ad1e74d4c6385c326e84a0f4080cf8134bb2842a Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Wed, 24 Aug 2022 13:06:05 -0400 Subject: [PATCH 6/8] select submitted model based on ModelType Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 42 +++++++++++++++++++---------------- monai/fl/utils/constants.py | 7 +++++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index 942c623a40..d0332a7935 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -29,6 +29,7 @@ FlStatistics, RequiredBundleKeys, WeightType, + ModelType ) from monai.fl.utils.exchange_object import ExchangeObject from monai.networks.utils import copy_model_state, get_state_dict @@ -101,7 +102,8 @@ class MonaiAlgo(ClientAlgo): config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json". config_filters_filename: filter configuration file. disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to True. - best_model_filepath: location of best model checkpoint (defaults "model/model.pt" relative to `bundle_root`) + model_filepaths: dict of locations of best & final model checkpoints; + defaults "model/model.pt" and "models/model_final.pt" relative to `bundle_root` for the best and final model, respectively. """ def __init__( @@ -113,7 +115,7 @@ def __init__( config_evaluate_filename: Optional[str] = "configs/evaluate.json", config_filters_filename: Optional[str] = None, disable_ckpt_loader: bool = True, - best_model_filepath: Optional[str] = "models/model.pt", + model_filepaths: Optional[dict] = {ModelType.BEST_MODEL: "models/model.pt", ModelType.FINAL_MODEL: "models/model_final.pt"} ): self.logger = logging.getLogger(self.__class__.__name__) @@ -124,7 +126,7 @@ def __init__( self.config_evaluate_filename = config_evaluate_filename self.config_filters_filename = config_filters_filename self.disable_ckpt_loader = disable_ckpt_loader - self.best_model_filepath = best_model_filepath + self.model_filepaths = model_filepaths self.app_root = None self.train_parser = None @@ -147,7 +149,7 @@ def initialize(self, extra=None): Args: extra: Dict with additional information that should be provided by FL system, - i.e., `ExtraItems.CLIENT_NAME` and `ExtraItems.APP_ROOT`. + i.e., `ExtraItems.CLIENT_NAME` and `ExtraItems.APP_ROOT`. """ if extra is None: @@ -263,27 +265,29 @@ def get_weights(self, extra=None): Returns: return_weights: `ExchangeObject` containing current weights (default) - or "best" local weights if `extra` has `ExtraItems.MODEL_NAME` with string containing "best". + or load requested model type from disk (`ModelType.BEST_MODEL` or `ModelType.FINAL_MODEL`). """ if extra is None: extra = {} - # by default return current weights, return best if requested via model name. - return_best = False - if ExtraItems.MODEL_NAME in extra: - if "best" in extra.get(ExtraItems.MODEL_NAME).lower(): - return_best = True + # by default return current weights, return best if requested via model type. self.phase = FlPhase.GET_WEIGHTS - - if return_best: - self.best_model_filepath = os.path.join(self.bundle_root, self.best_model_filepath) - if not os.path.isfile(self.best_model_filepath): - raise ValueError(f"No best model checkpoint exists at {self.best_model_filepath}") - weights = torch.load(self.best_model_filepath, map_location="cpu") - weigh_type = WeightType.WEIGHTS - stats = dict() - self.logger.info(f"Returning best checkpoint weights from {self.best_model_filepath}.") + + if ExtraItems.MODEL_TYPE in extra: + model_type = extra.get(ExtraItems.MODEL_TYPE) + if not isinstance(model_type, ModelType): + raise ValueError(f"Expected requested model type to be of type `ModelType` but received {type(model_type)}") + if model_type in self.model_filepaths: + model_path = os.path.join(self.bundle_root, self.model_filepaths[model_type]) + if not os.path.isfile(model_path): + raise ValueError(f"No best model checkpoint exists at {model_path}") + weights = torch.load(model_path, map_location="cpu") + weigh_type = WeightType.WEIGHTS + stats = dict() + self.logger.info(f"Returning best checkpoint weights from {model_path}.") + else: + raise ValueError(f"Requested model type {model_type} not specified in `model_filepahts`: {self.model_filepaths}") else: if self.trainer: weights = get_state_dict(self.trainer.network) diff --git a/monai/fl/utils/constants.py b/monai/fl/utils/constants.py index 2d3a8b28bc..f6da8d4ea0 100644 --- a/monai/fl/utils/constants.py +++ b/monai/fl/utils/constants.py @@ -17,9 +17,14 @@ class WeightType(StrEnum): WEIGHT_DIFF = "fl_weight_diff" +class ModelType(StrEnum): + BEST_MODEL = "fl_best_model" + FINAL_MODEL = "fl_final_model" + + class ExtraItems(StrEnum): ABORT = "fl_abort" - MODEL_NAME = "fl_model_name" + MODEL_TYPE = "fl_model_type" CLIENT_NAME = "fl_client_name" APP_ROOT = "fl_app_root" From c9d8a57533024e3b0996f0c0cebe92bb9e618237 Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Wed, 24 Aug 2022 13:25:18 -0400 Subject: [PATCH 7/8] support deterministic training Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 39 +++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index d0332a7935..5ffb12ff51 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -18,7 +18,7 @@ import monai from monai.bundle import ConfigParser -from monai.bundle.config_item import ConfigItem, ConfigComponent +from monai.bundle.config_item import ConfigComponent, ConfigItem from monai.config import IgniteInfo from monai.fl.client.client_algo import ClientAlgo from monai.fl.utils.constants import ( @@ -27,9 +27,9 @@ FiltersType, FlPhase, FlStatistics, + ModelType, RequiredBundleKeys, WeightType, - ModelType ) from monai.fl.utils.exchange_object import ExchangeObject from monai.networks.utils import copy_model_state, get_state_dict @@ -101,9 +101,15 @@ class MonaiAlgo(ClientAlgo): config_train_filename: bundle training config path relative to bundle_root; defaults to "configs/train.json". config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json". config_filters_filename: filter configuration file. - disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to True. + disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to `True`. model_filepaths: dict of locations of best & final model checkpoints; - defaults "model/model.pt" and "models/model_final.pt" relative to `bundle_root` for the best and final model, respectively. + defaults "model/model.pt" and "models/model_final.pt" relative to `bundle_root` + for the best and final model, respectively. + seed: set random seed for modules to enable or disable deterministic training; defaults to `None`, + i.e., non-deterministic training. + benchmark: set benchmark to `False` for full deterministic behavior in cuDNN components. + Note, full determinism in federated learning depends also on deterministic behavior of other FL components, + e.g., the aggregator, which is not controlled by this class. """ def __init__( @@ -115,7 +121,12 @@ def __init__( config_evaluate_filename: Optional[str] = "configs/evaluate.json", config_filters_filename: Optional[str] = None, disable_ckpt_loader: bool = True, - model_filepaths: Optional[dict] = {ModelType.BEST_MODEL: "models/model.pt", ModelType.FINAL_MODEL: "models/model_final.pt"} + model_filepaths: Optional[dict] = { + ModelType.BEST_MODEL: "models/model.pt", + ModelType.FINAL_MODEL: "models/model_final.pt", + }, + seed: Optional[int] = None, + benchmark: bool = True, ): self.logger = logging.getLogger(self.__class__.__name__) @@ -127,6 +138,8 @@ def __init__( self.config_filters_filename = config_filters_filename self.disable_ckpt_loader = disable_ckpt_loader self.model_filepaths = model_filepaths + self.seed = seed + self.benchmark = benchmark self.app_root = None self.train_parser = None @@ -154,8 +167,12 @@ def initialize(self, extra=None): """ if extra is None: extra = {} - self.logger.info(f"Initializing {self.client_name} ...") self.client_name = extra.get(ExtraItems.CLIENT_NAME, "noname") + self.logger.info(f"Initializing {self.client_name} ...") + + if self.seed: + monai.utils.set_determinism(seed=self.seed) + setattr(torch.backends.cudnn, "benchmark", self.benchmark) # FL platform needs to provide filepath to configuration files self.app_root = extra.get(ExtraItems.APP_ROOT, "") @@ -273,11 +290,13 @@ def get_weights(self, extra=None): # by default return current weights, return best if requested via model type. self.phase = FlPhase.GET_WEIGHTS - + if ExtraItems.MODEL_TYPE in extra: model_type = extra.get(ExtraItems.MODEL_TYPE) if not isinstance(model_type, ModelType): - raise ValueError(f"Expected requested model type to be of type `ModelType` but received {type(model_type)}") + raise ValueError( + f"Expected requested model type to be of type `ModelType` but received {type(model_type)}" + ) if model_type in self.model_filepaths: model_path = os.path.join(self.bundle_root, self.model_filepaths[model_type]) if not os.path.isfile(model_path): @@ -287,7 +306,9 @@ def get_weights(self, extra=None): stats = dict() self.logger.info(f"Returning best checkpoint weights from {model_path}.") else: - raise ValueError(f"Requested model type {model_type} not specified in `model_filepahts`: {self.model_filepaths}") + raise ValueError( + f"Requested model type {model_type} not specified in `model_filepahts`: {self.model_filepaths}" + ) else: if self.trainer: weights = get_state_dict(self.trainer.network) From 0c04b7d0b67427d4c33695d961f9769919720ac1 Mon Sep 17 00:00:00 2001 From: Holger Roth Date: Wed, 24 Aug 2022 13:32:36 -0400 Subject: [PATCH 8/8] fix formatting and input argument defaults Signed-off-by: Holger Roth --- monai/fl/client/monai_algo.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/monai/fl/client/monai_algo.py b/monai/fl/client/monai_algo.py index 5ffb12ff51..013fe0ed1b 100644 --- a/monai/fl/client/monai_algo.py +++ b/monai/fl/client/monai_algo.py @@ -102,9 +102,8 @@ class MonaiAlgo(ClientAlgo): config_evaluate_filename: bundle evaluation config path relative to bundle_root; defaults to "configs/evaluate.json". config_filters_filename: filter configuration file. disable_ckpt_loader: do not use CheckpointLoader if defined in train/evaluate configs; defaults to `True`. - model_filepaths: dict of locations of best & final model checkpoints; - defaults "model/model.pt" and "models/model_final.pt" relative to `bundle_root` - for the best and final model, respectively. + best_model_filepath: location of best model checkpoint; defaults "models/model.pt" relative to `bundle_root`. + final_model_filepath: location of final model checkpoint; defaults "models/model_final.pt" relative to `bundle_root`. seed: set random seed for modules to enable or disable deterministic training; defaults to `None`, i.e., non-deterministic training. benchmark: set benchmark to `False` for full deterministic behavior in cuDNN components. @@ -121,10 +120,8 @@ def __init__( config_evaluate_filename: Optional[str] = "configs/evaluate.json", config_filters_filename: Optional[str] = None, disable_ckpt_loader: bool = True, - model_filepaths: Optional[dict] = { - ModelType.BEST_MODEL: "models/model.pt", - ModelType.FINAL_MODEL: "models/model_final.pt", - }, + best_model_filepath: Optional[str] = "models/model.pt", + final_model_filepath: Optional[str] = "models/model_final.pt", seed: Optional[int] = None, benchmark: bool = True, ): @@ -137,7 +134,7 @@ def __init__( self.config_evaluate_filename = config_evaluate_filename self.config_filters_filename = config_filters_filename self.disable_ckpt_loader = disable_ckpt_loader - self.model_filepaths = model_filepaths + self.model_filepaths = {ModelType.BEST_MODEL: best_model_filepath, ModelType.FINAL_MODEL: final_model_filepath} self.seed = seed self.benchmark = benchmark @@ -172,7 +169,7 @@ def initialize(self, extra=None): if self.seed: monai.utils.set_determinism(seed=self.seed) - setattr(torch.backends.cudnn, "benchmark", self.benchmark) + torch.backends.cudnn.benchmark = self.benchmark # FL platform needs to provide filepath to configuration files self.app_root = extra.get(ExtraItems.APP_ROOT, "")