From 323db85f35352f4634fb2044bf0ad84dc5315cd6 Mon Sep 17 00:00:00 2001 From: Lev Kurilenko Date: Wed, 5 Oct 2022 00:41:25 +0000 Subject: [PATCH 1/3] Change type to tuple in replace_wo_policy isinstance check --- deepspeed/module_inject/replace_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index ea0e13726316..06c27a6445f9 100755 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -690,7 +690,7 @@ def _replace(child, name, conv_linear_layer): weight_shape = child.weight.ds_shape else: weight_shape = child.weight.shape - if isinstance(all_reduce_linears, dict) and name in all_reduce_linears: + if isinstance(all_reduce_linears, tuple) and name in all_reduce_linears: new_weight = torch.empty(( weight_shape[1] if conv_linear_layer else weight_shape[0], (weight_shape[0] if conv_linear_layer else weight_shape[1]) // From 843825d8857a79a1cea6d6bc944d5eeab6c795fa Mon Sep 17 00:00:00 2001 From: Lev Kurilenko Date: Thu, 6 Oct 2022 01:04:59 +0000 Subject: [PATCH 2/3] Add str type check as well for single value tuples that become strings --- deepspeed/module_inject/replace_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index 06c27a6445f9..63081e5a0254 100755 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -690,7 +690,7 @@ def _replace(child, name, conv_linear_layer): weight_shape = child.weight.ds_shape else: weight_shape = child.weight.shape - if isinstance(all_reduce_linears, tuple) and name in all_reduce_linears: + if (isinstance(all_reduce_linears, tuple) or isinstance(all_reduce_linears, str)) and name in all_reduce_linears: new_weight = torch.empty(( weight_shape[1] if conv_linear_layer else weight_shape[0], (weight_shape[0] if conv_linear_layer else weight_shape[1]) // From 73d41a055e952d35856cb8f377fec28ba9b5777c Mon Sep 17 00:00:00 2001 From: Lev Kurilenko Date: Fri, 7 Oct 2022 19:07:35 +0000 Subject: [PATCH 3/3] pre-commit --- deepspeed/module_inject/replace_module.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepspeed/module_inject/replace_module.py b/deepspeed/module_inject/replace_module.py index 63081e5a0254..30af889c34ac 100755 --- a/deepspeed/module_inject/replace_module.py +++ b/deepspeed/module_inject/replace_module.py @@ -690,7 +690,9 @@ def _replace(child, name, conv_linear_layer): weight_shape = child.weight.ds_shape else: weight_shape = child.weight.shape - if (isinstance(all_reduce_linears, tuple) or isinstance(all_reduce_linears, str)) and name in all_reduce_linears: + if (isinstance(all_reduce_linears, + tuple) or isinstance(all_reduce_linears, + str)) and name in all_reduce_linears: new_weight = torch.empty(( weight_shape[1] if conv_linear_layer else weight_shape[0], (weight_shape[0] if conv_linear_layer else weight_shape[1]) //