From d1cd2f62cb9ed1bca857c5c8ca4704157ba23e7c Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:26:17 +0800 Subject: [PATCH 01/11] Fixed G31 --- python/paddle/distributed/ps/utils/public.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/ps/utils/public.py b/python/paddle/distributed/ps/utils/public.py index b9fb44b57a245..a50eebc13af85 100755 --- a/python/paddle/distributed/ps/utils/public.py +++ b/python/paddle/distributed/ps/utils/public.py @@ -1354,7 +1354,7 @@ def insert_communicate_op( outputs={"Out": []}, attrs={ "mode": "forward" if is_forward else "backward", - "send_var_name": entrance_var + ["microbatch_id"], + "send_var_name": [*entrance_var, "microbatch_id"], "recv_var_name": [], "message_name": comm_info["block_input_var_name"], "next_endpoints": next_heter_worker_endpoints, From 421e488d2550ed774ccf76d0918035c013e248cb Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:27:32 +0800 Subject: [PATCH 02/11] Fixed G32 --- python/paddle/distributed/transpiler/collective.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/transpiler/collective.py b/python/paddle/distributed/transpiler/collective.py index b35bb29862c8e..56df3ed7a76bf 100644 --- a/python/paddle/distributed/transpiler/collective.py +++ b/python/paddle/distributed/transpiler/collective.py @@ -814,7 +814,7 @@ def _insert_allgather_ops(self): param = block.vars[op_role_var[i]] new_grad_var = block.create_var( name=op_role_var[i] + "_allgather", - shape=[self.allgather_ranks] + list(param.shape), + shape=[self.allgather_ranks, *list(param.shape)], persistable=False, dtype=core.VarDesc.VarType.FP32, stop_gradient=True, From 2e9e136db247afba544ad8c98dfda225b19e1a6e Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:28:52 +0800 Subject: [PATCH 03/11] Fixed G33 --- python/paddle/distributed/utils/launch_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/utils/launch_utils.py b/python/paddle/distributed/utils/launch_utils.py index fa1deedc0bbda..f6dfb65e1452f 100644 --- a/python/paddle/distributed/utils/launch_utils.py +++ b/python/paddle/distributed/utils/launch_utils.py @@ -466,7 +466,7 @@ def start_local_trainers( logger.debug(f"trainer proc env:{current_env}") - cmd = [sys.executable, "-u", training_script] + training_script_args + cmd = [sys.executable, "-u", training_script, *training_script_args] logger.info(f"start trainer proc:{cmd} env:{proc_env}") From c3e3ef2cd7bac4847aae996623cc097d75ba3d8a Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:31:06 +0800 Subject: [PATCH 04/11] Fixed G34 --- python/paddle/distribution/lkj_cholesky.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/paddle/distribution/lkj_cholesky.py b/python/paddle/distribution/lkj_cholesky.py index 1a158a79c3cf4..1b196dcdc99b2 100644 --- a/python/paddle/distribution/lkj_cholesky.py +++ b/python/paddle/distribution/lkj_cholesky.py @@ -73,7 +73,8 @@ def vec_to_tril_matrix( # Calculate the dimension of the square matrix based on the last but one dimension of `p` # Define the output shape, which adds two dimensions for the square matrix shape0 = flatten_shape // last_dim - output_shape = sample_shape + ( + output_shape = ( + *sample_shape, shape0 // reduce(operator.mul, sample_shape), dim, dim, @@ -251,7 +252,7 @@ def _onion(self, sample_shape: Sequence[int]) -> Tensor: # u_hypersphere[..., 0, :].fill_(0.0) # u_hypersphere[..., 0, :] = 0.0 u_hypersphere_other = u_hypersphere[..., 1:, :] - zero_shape = tuple(u_hypersphere.shape[:-2]) + (1, self.dim) + zero_shape = (*tuple(u_hypersphere.shape[:-2]), 1, self.dim) zero_row = paddle.zeros(shape=zero_shape, dtype=u_hypersphere.dtype) u_hypersphere = paddle.concat([zero_row, u_hypersphere_other], axis=-2) From 1472d57871a42d606d9084891887198af0ec4058 Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:34:16 +0800 Subject: [PATCH 05/11] Fixed G35 --- python/paddle/distribution/multinomial.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/paddle/distribution/multinomial.py b/python/paddle/distribution/multinomial.py index c233fc8c7dc32..d0be2d9a842c6 100644 --- a/python/paddle/distribution/multinomial.py +++ b/python/paddle/distribution/multinomial.py @@ -159,12 +159,7 @@ def sample(self, shape: Iterable[int] = ()) -> Tensor: if not isinstance(shape, Iterable): raise TypeError('sample shape must be Iterable object.') - samples = self._categorical.sample( - [ - self.total_count, - ] - + list(shape) - ) + samples = self._categorical.sample([self.total_count, *list(shape)]) return ( paddle.nn.functional.one_hot(samples, self.probs.shape[-1]) .cast(self.probs.dtype) From e8d929fdafdd97d3d3b873458dbaee178a622908 Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:37:24 +0800 Subject: [PATCH 06/11] Fixed G36 --- .../distribution/multivariate_normal.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/paddle/distribution/multivariate_normal.py b/python/paddle/distribution/multivariate_normal.py index 9f5f27931d1e8..23878e2a6b0d5 100644 --- a/python/paddle/distribution/multivariate_normal.py +++ b/python/paddle/distribution/multivariate_normal.py @@ -133,7 +133,7 @@ def __init__( scale_tril.shape[:-2], loc.shape[:-1] ) self.scale_tril = scale_tril.expand( - batch_shape + [scale_tril.shape[-2], scale_tril.shape[-1]] + *batch_shape, scale_tril.shape[-2], scale_tril.shape[-1] ) elif covariance_matrix is not None: if covariance_matrix.dim() < 2: @@ -146,8 +146,9 @@ def __init__( covariance_matrix.shape[:-2], loc.shape[:-1] ) self.covariance_matrix = covariance_matrix.expand( - batch_shape - + [covariance_matrix.shape[-2], covariance_matrix.shape[-1]] + *batch_shape, + covariance_matrix.shape[-2], + covariance_matrix.shape[-1], ) else: if precision_matrix.dim() < 2: @@ -160,15 +161,11 @@ def __init__( precision_matrix.shape[:-2], loc.shape[:-1] ) self.precision_matrix = precision_matrix.expand( - batch_shape - + [precision_matrix.shape[-2], precision_matrix.shape[-1]] + *batch_shape, + precision_matrix.shape[-2], + precision_matrix.shape[-1], ) - self.loc = loc.expand( - batch_shape - + [ - -1, - ] - ) + self.loc = loc.expand([*batch_shape, -1]) event_shape = self.loc.shape[-1:] if scale_tril is not None: From c39ae9d91cc3f579df1c43c8e90ca431326cacc2 Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:39:34 +0800 Subject: [PATCH 07/11] Fixed G37 --- .../distributed/auto_parallel/static/callbacks.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/paddle/distributed/auto_parallel/static/callbacks.py b/python/paddle/distributed/auto_parallel/static/callbacks.py index 8b2d65122d121..e8c4e7a7bf2e4 100644 --- a/python/paddle/distributed/auto_parallel/static/callbacks.py +++ b/python/paddle/distributed/auto_parallel/static/callbacks.py @@ -45,19 +45,19 @@ def config_callbacks( cbks = cbks if isinstance(cbks, (list, tuple)) else [cbks] if not any(isinstance(k, ProgBarLogger) for k in cbks) and verbose: - cbks = [ProgBarLoggerAuto(log_freq, verbose=verbose)] + cbks + cbks = [ProgBarLoggerAuto(log_freq, verbose=verbose), *cbks] if not any(isinstance(k, LRScheduler) for k in cbks): - cbks = [LRSchedulerAuto()] + cbks + cbks = [LRSchedulerAuto(), *cbks] if not any(isinstance(k, ModelCheckpoint) for k in cbks): - cbks = cbks + [ModelCheckpointAuto(save_freq, save_dir)] + cbks = [*cbks, ModelCheckpointAuto(save_freq, save_dir)] if not any(isinstance(k, Profiler) for k in cbks) and verbose == 3: - cbks = cbks + [Profiler(timer_only=True)] + cbks = [*cbks, Profiler(timer_only=True)] if not any(isinstance(k, History) for k in cbks): - cbks = cbks + [History()] + cbks = [*cbks, History()] for i, k in enumerate(cbks): if isinstance(k, ProgBarLogger): From ba473a5a9f0a196707abd557aaac986e92651a9f Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:40:23 +0800 Subject: [PATCH 08/11] Fixed G38 --- .../fleet/parameter_server/distribute_transpiler/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py index bc9c6c4ec7c28..1e8de775acd53 100644 --- a/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py +++ b/python/paddle/incubate/distributed/fleet/parameter_server/distribute_transpiler/__init__.py @@ -554,7 +554,7 @@ def _save_dense_params(self, executor, dirname, context, main_program): optimizer.type, varname ) - for var_name in [varname] + reshaped_varnames + origin_varnames: + for var_name in [varname, *reshaped_varnames, *origin_varnames]: var = self._origin_main_program.global_block().vars[var_name] block.append_op( type='recv_save', From eadb45003bd867800e07189ddd806aab46377575 Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:41:20 +0800 Subject: [PATCH 09/11] Fixed G39 --- .../distributed/fleet/parameter_server/ir/pserver_pass.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/incubate/distributed/fleet/parameter_server/ir/pserver_pass.py b/python/paddle/incubate/distributed/fleet/parameter_server/ir/pserver_pass.py index a2f81b17fbc78..9f63499db5484 100644 --- a/python/paddle/incubate/distributed/fleet/parameter_server/ir/pserver_pass.py +++ b/python/paddle/incubate/distributed/fleet/parameter_server/ir/pserver_pass.py @@ -904,7 +904,7 @@ def add_large_scale_op( names_str = ",".join(value_names) dims_str = ",".join([str(dim) for dim in value_dims]) ids_name = f"kSparseIDs@{param}" - cached_str = ",".join(acture_names + [ids_name]) + cached_str = ",".join([*acture_names, ids_name]) init_attr_str = get_initializer_attrs(acture_names) meta_str = ":".join( From 5e4dec3a7cc86b16f1fa456c67911e903b85cb2e Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:42:29 +0800 Subject: [PATCH 10/11] Fixed G40 --- .../distributed/fleet/parameter_server/ir/trainer_pass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/incubate/distributed/fleet/parameter_server/ir/trainer_pass.py b/python/paddle/incubate/distributed/fleet/parameter_server/ir/trainer_pass.py index 5578c991a2b90..e97979d6937a2 100644 --- a/python/paddle/incubate/distributed/fleet/parameter_server/ir/trainer_pass.py +++ b/python/paddle/incubate/distributed/fleet/parameter_server/ir/trainer_pass.py @@ -1264,7 +1264,7 @@ def insert_communicate_op( outputs={"Out": []}, attrs={ "mode": "forward" if is_forward else "backward", - "send_var_name": entrance_var + ["microbatch_id"], + "send_var_name": [*entrance_var, "microbatch_id"], "recv_var_name": [], "message_name": comm_info["block_input_var_name"], "next_endpoints": next_heter_worker_endpoints, @@ -1339,7 +1339,7 @@ def replace_ops_by_communicate_op( outputs={"Out": []}, attrs={ "mode": "forward", - "send_var_name": entrance_var + ["microbatch_id"], + "send_var_name": [*entrance_var, "microbatch_id"], "recv_var_name": [], "message_name": comm_info["block_input_var_name"], "next_endpoints": next_heter_worker_endpoints, From ae9093414ad07dab2632dde0d9c348a78b15e1ca Mon Sep 17 00:00:00 2001 From: MufanColin <76479709+MufanColin@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:00:30 +0800 Subject: [PATCH 11/11] Try to fix errors --- .../paddle/distribution/multivariate_normal.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/python/paddle/distribution/multivariate_normal.py b/python/paddle/distribution/multivariate_normal.py index 23878e2a6b0d5..948cc9d87ffe1 100644 --- a/python/paddle/distribution/multivariate_normal.py +++ b/python/paddle/distribution/multivariate_normal.py @@ -133,7 +133,7 @@ def __init__( scale_tril.shape[:-2], loc.shape[:-1] ) self.scale_tril = scale_tril.expand( - *batch_shape, scale_tril.shape[-2], scale_tril.shape[-1] + [*batch_shape, scale_tril.shape[-2], scale_tril.shape[-1]] ) elif covariance_matrix is not None: if covariance_matrix.dim() < 2: @@ -146,9 +146,11 @@ def __init__( covariance_matrix.shape[:-2], loc.shape[:-1] ) self.covariance_matrix = covariance_matrix.expand( - *batch_shape, - covariance_matrix.shape[-2], - covariance_matrix.shape[-1], + [ + *batch_shape, + covariance_matrix.shape[-2], + covariance_matrix.shape[-1], + ] ) else: if precision_matrix.dim() < 2: @@ -161,9 +163,11 @@ def __init__( precision_matrix.shape[:-2], loc.shape[:-1] ) self.precision_matrix = precision_matrix.expand( - *batch_shape, - precision_matrix.shape[-2], - precision_matrix.shape[-1], + [ + *batch_shape, + precision_matrix.shape[-2], + precision_matrix.shape[-1], + ] ) self.loc = loc.expand([*batch_shape, -1]) event_shape = self.loc.shape[-1:]