[CodeStyle][F401] remove unused imports in python/paddle/distributed

python/paddle/distributed/auto_parallel/cluster_v2.py

@@ -12,15 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
 import numpy as np
 from enum import IntEnum
 from enum import unique
 
-import paddle
 from paddle.fluid import core
-from paddle.fluid.core import Device
-from paddle.fluid.core import Link
+from paddle.fluid.core import Device  # noqa: F401
+from paddle.fluid.core import Link  # noqa: F401
 
 
 @unique

python/paddle/distributed/auto_parallel/completion.py

@@ -13,17 +13,13 @@
 # limitations under the License.
 
 import copy
-from copy import deepcopy
 import time
 
 from paddle.fluid import core
-from paddle.fluid import framework
 
-from .utils import print_program_with_dist_attr, is_gradient_clip_op
+from .utils import is_gradient_clip_op
 from .operators import find_compatible_distributed_operator_impls
-from .dist_context import get_default_distributed_context, _node_id
-from .dist_tensor import DistributedTensor
-from .dist_op import DistributedOperator
+from .dist_context import _node_id
 from .dist_attribute import TensorDistributedAttribute
 from .dist_attribute import OperatorDistributedAttribute
 from .process_mesh import ProcessMesh

python/paddle/distributed/auto_parallel/cost/base_cost.py

@@ -17,7 +17,7 @@
 
 import paddle
 
-from ..utils import _get_comm_group, _get_corresponding_rank
+from ..utils import _get_comm_group
 from ..process_group import get_process_group
 from ..cluster import LinkType
 from ..dist_tensor import DistributedTensor

python/paddle/distributed/auto_parallel/cost/comm_op_cost.py

@@ -14,7 +14,7 @@
 
 import math
 
-from .base_cost import register_op_cost, CommOpCost, _g_op_cost_factory
+from .base_cost import CommOpCost, register_op_cost
 
 
 @register_op_cost

python/paddle/distributed/auto_parallel/cost/comp_op_cost.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License
 
-from .base_cost import Cost, register_op_cost, CompOpCost, _g_op_cost_factory
+from .base_cost import CompOpCost, register_op_cost
 
 
 @register_op_cost

python/paddle/distributed/auto_parallel/cost/estimate_cost.py

@@ -16,7 +16,6 @@
 from functools import reduce
 
 import paddle
-import paddle.fluid.core as core
 from paddle.distributed.fleet.meta_optimizers.common import OpRole
 
 from .base_cost import Cost

python/paddle/distributed/auto_parallel/cost_model.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import json
 import queue
 import copy
 from enum import Enum

python/paddle/distributed/auto_parallel/dist_attribute.py

@@ -13,7 +13,6 @@
 # limitations under the License
 
 import copy
-from collections import defaultdict
 from paddle.fluid.framework import Variable
 from .process_mesh import ProcessMesh
 

python/paddle/distributed/auto_parallel/dist_context.py

@@ -14,17 +14,14 @@
 
 import copy
 from collections import defaultdict
-import paddle.fluid
 from paddle.fluid import framework
-from paddle.fluid.framework import get_flags, set_flags
+from paddle.fluid.framework import set_flags
 from paddle.fluid import core
 from paddle.distributed.passes import PassContext
-from .dist_attribute import TensorDistributedAttribute
-from .dist_attribute import OperatorDistributedAttribute
 from .dist_tensor import DistributedTensor
 from .dist_op import DistributedOperator
 from .process_mesh import ProcessMesh
-from .utils import is_loss_grad_op, is_loss_op
+from .utils import is_loss_grad_op
 
 # There always exists a default context for user. And user can set it to another one.
 _g_default_distributed_context = None

python/paddle/distributed/auto_parallel/dist_loader.py

@@ -14,12 +14,9 @@
 
 import abc
 import numpy as np
-from functools import wraps
 
 import paddle
-from .utils import to_list
-from paddle.fluid.layers.utils import flatten
-from paddle.io import DataLoader, BatchSampler, IterableDataset
+from paddle.io import BatchSampler, IterableDataset
 from paddle.fluid.dataloader.batch_sampler import _InfiniteIterableSampler
 from paddle.fluid.dataloader.dataloader_iter import _DatasetKind, default_collate_fn, default_convert_fn
 

python/paddle/distributed/auto_parallel/dist_op.py

@@ -13,16 +13,12 @@
 # limitations under the License
 
 import copy
-from collections import defaultdict
 import paddle
 from paddle.fluid import core
 from paddle.fluid.framework import Variable
-from .dist_attribute import TensorDistributedAttribute
 from .dist_attribute import OperatorDistributedAttribute
 from .dist_attribute import append_op_input_suffix
 from .dist_attribute import append_op_output_suffix
-from .dist_attribute import get_tensor_dist_attr_field_keys
-from .dist_attribute import get_op_dist_attr_field_keys
 from .utils import convert_to_shard_spec, verify_shard_spec
 
 

python/paddle/distributed/auto_parallel/dist_saver.py

@@ -16,16 +16,13 @@
 import os
 import errno
 import pickle
-import warnings
 import logging
 import numpy as np
 import paddle
 
 from paddle import fluid
 from paddle.fluid import core
-from paddle.fluid.framework import static_only
 from .utils import get_dist_attr
-from .converter import Converter
 from .process_group import _g_process_group_map
 from ..utils.log_utils import get_logger
 

python/paddle/distributed/auto_parallel/dist_tensor.py

@@ -19,7 +19,6 @@
 from paddle.fluid import core
 from paddle.fluid.framework import Parameter, Block, Variable
 from .dist_attribute import TensorDistributedAttribute
-from .dist_attribute import get_tensor_dist_attr_field_keys
 from .utils import _linear_idx2coordinate
 
 

python/paddle/distributed/auto_parallel/engine.py

@@ -13,8 +13,6 @@
 # limitations under the License.
 
 import os
-import time
-import copy
 import logging
 import random
 import numpy as np
@@ -24,14 +22,13 @@
 import paddle.utils as utils
 
 from paddle import fluid, profiler, static
-from paddle.jit import to_static
 from paddle.metric import Metric
 from paddle.static import InputSpec
 from paddle.fluid import core
 from paddle.fluid import Variable
 from paddle.fluid.layers.utils import flatten
 from paddle.fluid.executor import global_scope, _to_name_str
-from paddle.fluid.framework import Operator, Parameter, _non_static_mode
+from paddle.fluid.framework import Operator, _non_static_mode
 from paddle.fluid.framework import _current_expected_place as _get_device
 from paddle.fluid.dygraph.parallel import ParallelEnv
 from paddle.distributed import fleet
@@ -44,7 +41,7 @@
 from .dist_op import DistributedOperator
 from .dist_saver import DistributedSaver
 from .dist_loader import NonIterableGeneratorLoader
-from .utils import print_program_with_dist_attr, to_list
+from .utils import to_list
 from .utils import get_logger, get_dist_attr
 from .process_group import new_process_group, get_all_process_groups
 from .dist_context import DistributedContext, get_default_distributed_context

python/paddle/distributed/auto_parallel/helper.py

@@ -15,11 +15,9 @@
 import logging
 from collections import defaultdict
 
-import paddle
-
 from paddle.nn import Layer
 from paddle.jit import to_static, not_to_static
-from paddle.fluid.framework import Operator, Parameter, _non_static_mode
+from paddle.fluid.framework import Parameter
 from paddle.fluid.framework import program_guard
 from paddle.fluid.executor import global_scope
 from paddle.fluid.dygraph.dygraph_to_static.program_translator import StaticFunction

python/paddle/distributed/auto_parallel/interface.py

@@ -12,14 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections import defaultdict
-
 import paddle
 from paddle.fluid import core
 from .process_mesh import ProcessMesh
 from .process_mesh import get_current_process_mesh
-from .process_mesh import set_current_process_mesh
-from .process_mesh import reset_current_process_mesh
 from .dist_context import get_default_distributed_context
 from .dist_tensor import DistributedTensor
 from .dist_op import DistributedOperatorHelper

python/paddle/distributed/auto_parallel/mapper.py

@@ -15,11 +15,8 @@
 import os
 import operator
 import functools
-import json
 import paddle
 from collections import deque
-from .graph import Node
-from .graph import Edge
 from .graph import Graph
 from .cluster import DeviceType
 from .process_group import get_process_group

python/paddle/distributed/auto_parallel/operators/common.py

@@ -13,8 +13,7 @@
 # limitations under the License
 
 import abc
-import paddle
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
+from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole
 from ..dist_attribute import OperatorDistributedAttribute
 from ..utils import _get_comm_group, _get_corresponding_rank, is_optimize_op
 from ..process_group import new_process_group

python/paddle/distributed/auto_parallel/operators/dist_check_finite_and_unscale.py

@@ -16,10 +16,8 @@
 from .common import DistributedOperatorImpl
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl
-from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
+from paddle.fluid import core
+from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole
 from ..utils import set_var_dist_attr
 from ..utils import set_dist_op_desc_original_id
 from ..process_group import new_process_group

python/paddle/distributed/auto_parallel/operators/dist_default.py

@@ -17,19 +17,11 @@
 from .common import register_distributed_operator_impl_container
 from .common import gradient_synchronization
 from .common import register_distributed_operator_impl, is_parameter_related
-from ..utils import is_dim_shard
-from ..utils import is_dim_replicate
-from ..utils import is_valid_list_index, is_prim_op
+from ..utils import is_prim_op
 from ..utils import compute_compatible_dim_mapping
-from ..utils import compute_compatible_dims_mapping
-from ..utils import compute_compatible_and_update_dim_mapping
 from ..utils import set_dist_op_desc_original_id
 from ..dist_attribute import OperatorDistributedAttribute
-from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.framework import Program, Parameter, Variable, program_guard
-from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
+from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole
 from ..process_group import new_process_group
 from ..utils import _get_comm_group, _get_corresponding_rank
 from ..cost import _g_op_cost_factory

python/paddle/distributed/auto_parallel/operators/dist_eltwise.py

@@ -17,20 +17,9 @@
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl, is_parameter_related
 from .common import is_elementwise_op
-from ..utils import is_dim_shard
-from ..utils import is_dim_replicate
-from ..utils import is_valid_list_index
 from ..utils import compute_compatible_dim_mapping
 from ..utils import compute_compatible_dims_mapping
-from ..utils import compute_compatible_and_update_dim_mapping
-from ..dist_attribute import OperatorDistributedAttribute
-from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.framework import Program, Parameter, Variable, program_guard
-from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
-from ..process_group import new_process_group
-from ..utils import _get_comm_group, _get_corresponding_rank
+from paddle.distributed.fleet.meta_optimizers.common import OpRole
 from .dist_default import DistributedDefaultImpl0
 from ..cost import _g_op_cost_factory
 from ..cost import build_comp_desc_from_dist_op, build_dp_costs

python/paddle/distributed/auto_parallel/operators/dist_embedding.py

@@ -17,19 +17,14 @@
 from .common import DistributedOperatorImpl
 from .common import register_distributed_operator_impl_container
 from .common import gradient_synchronization
-from .common import register_distributed_operator_impl, set_comm_op_dist_attr_for_program, naive_copy_op_dist_attr_for_program, is_parameter_related
+from .common import naive_copy_op_dist_attr_for_program, register_distributed_operator_impl, set_comm_op_dist_attr_for_program
 from ..utils import is_dim_shard
 from ..utils import is_dim_replicate
-from ..utils import is_valid_list_index
-from ..utils import compute_compatible_dim_mapping
-from ..utils import compute_compatible_dims_mapping
 from ..utils import compute_compatible_and_update_dim_mapping
-from ..dist_attribute import OperatorDistributedAttribute, TensorDistributedAttribute
+from ..dist_attribute import OperatorDistributedAttribute
 from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.framework import Program, Parameter, Variable
 from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
+from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole
 from ..process_group import new_process_group
 from ..utils import _get_comm_group, _get_idx_in_axis, _get_corresponding_rank, set_var_dist_attr
 from ..cost import build_comp_desc_from_dist_op, build_comm_desc_from_dist_op

python/paddle/distributed/auto_parallel/operators/dist_fill_constant_batch_size_like.py

@@ -16,23 +16,12 @@
 from .common import DistributedOperatorImpl
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl
-from ..utils import is_dim_shard
-from ..utils import is_dim_replicate
-from ..utils import is_valid_list_index
-from ..utils import compute_compatible_dim_mapping
-from ..utils import compute_compatible_dims_mapping
 from ..utils import compute_compatible_and_update_dim_mapping
-from ..utils import set_dist_op_desc_original_id
-from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.framework import Program, Parameter, Variable, program_guard
-from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
 from paddle.distributed.fleet.meta_optimizers.common import OpRole
 from .dist_default import DistributedDefaultImpl0
 from ..cost import FillConstantBatchSizeLikeOpCost
-from ..cost import build_comp_desc_from_dist_op, build_dp_costs
+from ..cost import build_comp_desc_from_dist_op
 from ..cost import build_comp_costs_from_descs
-from paddle.distributed.auto_parallel.cost.comm_op_cost import AllreduceSumOpCost
 
 
 class DistributedFillConstantBatchSizeLike(DistributedOperatorImplContainer):

python/paddle/distributed/auto_parallel/operators/dist_fused_attention.py

@@ -17,9 +17,6 @@
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl
 from ..utils import is_dim_shard, is_dim_replicate
-from ..utils import is_valid_list_index
-from ..utils import compute_compatible_dim_mapping
-from ..utils import compute_compatible_dims_mapping
 from ..utils import compute_compatible_and_update_dim_mapping
 from .dist_default import DistributedDefaultImpl0
 from ..utils import _get_comm_group, _get_corresponding_rank

python/paddle/distributed/auto_parallel/operators/dist_fused_feedforward.py

@@ -17,9 +17,6 @@
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl
 from ..utils import is_dim_shard, is_dim_replicate
-from ..utils import is_valid_list_index
-from ..utils import compute_compatible_dim_mapping
-from ..utils import compute_compatible_dims_mapping
 from ..utils import compute_compatible_and_update_dim_mapping
 from .dist_default import DistributedDefaultImpl0
 from ..utils import _get_comm_group, _get_corresponding_rank

python/paddle/distributed/auto_parallel/operators/dist_matmul.py

@@ -20,20 +20,17 @@
 from .common import register_distributed_operator_impl_container
 from .common import register_distributed_operator_impl
 from .common import gradient_synchronization
-from .common import set_comm_op_dist_attr_for_program, naive_copy_op_dist_attr_for_program, is_parameter_related
+from .common import is_parameter_related, set_comm_op_dist_attr_for_program
 from ..utils import is_dim_shard
 from ..utils import is_dim_replicate
 from ..utils import is_valid_list_index
-from ..utils import compute_compatible_dim_mapping
 from ..utils import compute_compatible_dims_mapping
 from ..utils import compute_compatible_and_update_dim_mapping
 from ..utils import set_dist_op_desc_original_id
 from ..dist_attribute import OperatorDistributedAttribute
 from paddle.fluid import core, unique_name
-from paddle.fluid.framework import _non_static_mode
-from paddle.fluid.framework import Program, Parameter, Variable, program_guard
 from paddle.fluid.data_feeder import check_variable_and_dtype, check_dtype
-from paddle.distributed.fleet.meta_optimizers.common import OpRole, OP_ROLE_KEY, OP_ROLE_VAR_KEY
+from paddle.distributed.fleet.meta_optimizers.common import OP_ROLE_KEY, OpRole
 from ..process_group import new_process_group
 from ..utils import _get_comm_group, _get_corresponding_rank
 from .dist_default import DistributedDefaultImpl0