[CodeStyle][py2] remove six package (part1)

paddle/fluid/inference/tests/api/full_pascalvoc_test_preprocess.py

@@ -19,7 +19,7 @@
 import sys
 from paddle.dataset.common import download
 import tarfile
-from six.moves import StringIO
+from io import StringIO
 import hashlib
 import tarfile
 import argparse

python/paddle/dataset/cifar.py

@@ -31,8 +31,7 @@
 import paddle.dataset.common
 import paddle.utils.deprecated as deprecated
 import tarfile
-import six
-from six.moves import cPickle as pickle
+import pickle
 
 __all__ = []
 
@@ -46,11 +45,10 @@
 def reader_creator(filename, sub_name, cycle=False):
 
     def read_batch(batch):
-        data = batch[six.b('data')]
-        labels = batch.get(six.b('labels'), batch.get(six.b('fine_labels'),
-                                                      None))
+        data = batch[b'data']
+        labels = batch.get(b'labels', batch.get(b'fine_labels', None))
         assert labels is not None
-        for sample, label in six.moves.zip(data, labels):
+        for sample, label in zip(data, labels):
             yield (sample / 255.0).astype(numpy.float32), int(label)
 
     def reader():

python/paddle/dataset/common.py

@@ -20,7 +20,7 @@
 import sys
 import importlib
 import paddle.dataset
-import six.moves.cPickle as pickle
+import pickle
 import tempfile
 import glob
 import paddle

python/paddle/dataset/conll05.py

@@ -24,7 +24,6 @@
 import gzip
 import paddle.dataset.common
 import paddle.utils.deprecated as deprecated
-from six.moves import zip, range
 
 __all__ = []
 

python/paddle/dataset/image.py

@@ -30,39 +30,34 @@
   be keep consistent between the training and inference period.
 """
 
-import six
 import numpy as np
 # FIXME(minqiyang): this is an ugly fix for the numpy bug reported here
 # https://github.com/numpy/numpy/issues/12497
-if six.PY3:
-    import subprocess
-    import sys
-    import os
-    interpreter = sys.executable
-    # Note(zhouwei): if use Python/C 'PyRun_SimpleString', 'sys.executable'
-    # will be the C++ execubable on Windows
-    if sys.platform == 'win32' and 'python.exe' not in interpreter:
-        interpreter = sys.exec_prefix + os.sep + 'python.exe'
-    import_cv2_proc = subprocess.Popen([interpreter, "-c", "import cv2"],
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-    out, err = import_cv2_proc.communicate()
-    retcode = import_cv2_proc.poll()
-    if retcode != 0:
-        cv2 = None
-    else:
-        try:
-            import cv2
-        except ImportError:
-            cv2 = None
+import subprocess
+import sys
+import os
+
+interpreter = sys.executable
+# Note(zhouwei): if use Python/C 'PyRun_SimpleString', 'sys.executable'
+# will be the C++ execubable on Windows
+if sys.platform == 'win32' and 'python.exe' not in interpreter:
+    interpreter = sys.exec_prefix + os.sep + 'python.exe'
+import_cv2_proc = subprocess.Popen([interpreter, "-c", "import cv2"],
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+out, err = import_cv2_proc.communicate()
+retcode = import_cv2_proc.poll()
+if retcode != 0:
+    cv2 = None
 else:
     try:
         import cv2
     except ImportError:
         cv2 = None
+
 import os
 import tarfile
-import six.moves.cPickle as pickle
+import pickle
 
 __all__ = []
 

python/paddle/dataset/imdb.py

@@ -26,7 +26,6 @@
 import tarfile
 import re
 import string
-import six
 
 __all__ = []
 
@@ -49,9 +48,8 @@ def tokenize(pattern):
         while tf != None:
             if bool(pattern.match(tf.name)):
                 # newline and punctuations removal and ad-hoc tokenization.
-                yield tarf.extractfile(tf).read().rstrip(
-                    six.b("\n\r")).translate(None, six.b(
-                        string.punctuation)).lower().split()
+                yield tarf.extractfile(tf).read().rstrip(b'\n\r').translate(
+                    None, string.punctuation.encode('latin-1')).lower().split()
             tf = tarf.next()
 
 
@@ -66,11 +64,11 @@ def build_dict(pattern, cutoff):
             word_freq[word] += 1
 
     # Not sure if we should prune less-frequent words here.
-    word_freq = [x for x in six.iteritems(word_freq) if x[1] > cutoff]
+    word_freq = [x for x in word_freq.items() if x[1] > cutoff]
 
     dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0]))
     words, _ = list(zip(*dictionary))
-    word_idx = dict(list(zip(words, six.moves.range(len(words)))))
+    word_idx = dict(list(zip(words, range(len(words)))))
     word_idx['<unk>'] = len(words)
     return word_idx
 

python/paddle/dataset/imikolov.py

@@ -23,7 +23,6 @@
 import paddle.utils.deprecated as deprecated
 import collections
 import tarfile
-import six
 
 __all__ = []
 
@@ -68,13 +67,11 @@ def build_dict(min_word_freq=50):
             # remove <unk> for now, since we will set it as last index
             del word_freq['<unk>']
 
-        word_freq = [
-            x for x in six.iteritems(word_freq) if x[1] > min_word_freq
-        ]
+        word_freq = [x for x in word_freq.items() if x[1] > min_word_freq]
 
         word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0]))
         words, _ = list(zip(*word_freq_sorted))
-        word_idx = dict(list(zip(words, six.moves.range(len(words)))))
+        word_idx = dict(list(zip(words, range(len(words)))))
         word_idx['<unk>'] = len(words)
 
     return word_idx
@@ -96,7 +93,7 @@ def reader():
                     l = ['<s>'] + l.strip().split() + ['<e>']
                     if len(l) >= n:
                         l = [word_idx.get(w, UNK) for w in l]
-                        for i in six.moves.range(n, len(l) + 1):
+                        for i in range(n, len(l) + 1):
                             yield tuple(l[i - n:i])
                 elif DataType.SEQ == data_type:
                     l = l.strip().split()

python/paddle/dataset/mnist.py

@@ -23,7 +23,6 @@
 import gzip
 import numpy
 import struct
-from six.moves import range
 
 __all__ = []
 

python/paddle/dataset/movielens.py

@@ -28,7 +28,6 @@
 import paddle.utils.deprecated as deprecated
 import re
 import functools
-import six
 
 __all__ = []
 
@@ -206,7 +205,7 @@ def max_movie_id():
     Get the maximum value of movie id.
     """
     __initialize_meta_info__()
-    return six.moves.reduce(__max_index_info__, list(MOVIE_INFO.values())).index
+    return functools.reduce(__max_index_info__, list(MOVIE_INFO.values())).index
 
 
 @deprecated(
@@ -219,7 +218,7 @@ def max_user_id():
     Get the maximum value of user id.
     """
     __initialize_meta_info__()
-    return six.moves.reduce(__max_index_info__, list(USER_INFO.values())).index
+    return functools.reduce(__max_index_info__, list(USER_INFO.values())).index
 
 
 def __max_job_id_impl__(a, b):
@@ -239,7 +238,7 @@ def max_job_id():
     Get the maximum value of job id.
     """
     __initialize_meta_info__()
-    return six.moves.reduce(__max_job_id_impl__,
+    return functools.reduce(__max_job_id_impl__,
                             list(USER_INFO.values())).job_id
 
 

python/paddle/dataset/uci_housing.py

@@ -20,7 +20,6 @@
 """
 
 import numpy as np
-import six
 import tempfile
 import tarfile
 import os
@@ -75,7 +74,7 @@ def load_data(filename, feature_num=14, ratio=0.8):
         axis=0), data.sum(axis=0) / data.shape[0]
     # if you want to print the distribution of input data, you could use function of feature_range
     #feature_range(maximums[:-1], minimums[:-1])
-    for i in six.moves.range(feature_num - 1):
+    for i in range(feature_num - 1):
         data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
     offset = int(data.shape[0] * ratio)
     UCI_TRAIN_DATA = data[:offset]

python/paddle/dataset/wmt14.py

@@ -20,7 +20,6 @@
 
 """
 
-import six
 import tarfile
 
 import paddle.dataset.common
@@ -173,8 +172,8 @@ def get_dict(dict_size, reverse=True):
     tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN)
     src_dict, trg_dict = __read_to_dict(tar_file, dict_size)
     if reverse:
-        src_dict = {v: k for k, v in six.iteritems(src_dict)}
-        trg_dict = {v: k for k, v in six.iteritems(trg_dict)}
+        src_dict = {v: k for k, v in src_dict.items()}
+        trg_dict = {v: k for k, v in trg_dict.items()}
     return src_dict, trg_dict
 
 

python/paddle/dataset/wmt16.py

@@ -29,7 +29,6 @@
 """
 
 import os
-import six
 import tarfile
 from collections import defaultdict
 
@@ -63,9 +62,7 @@ def __build_dict(tar_file, dict_size, save_path, lang):
     with open(save_path, "wb") as fout:
         fout.write(("%s\n%s\n%s\n" % (START_MARK, END_MARK, UNK_MARK)).encode())
         for idx, word in enumerate(
-                sorted(six.iteritems(word_dict),
-                       key=lambda x: x[1],
-                       reverse=True)):
+                sorted(word_dict.items(), key=lambda x: x[1], reverse=True)):
             if idx + 3 == dict_size: break
             fout.write(word[0].encode())
             fout.write(b'\n')

python/paddle/distributed/elastic.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import argparse
-import six
 import os
 
 
@@ -30,7 +29,7 @@ def __init__(self, server, name):
         self.np_path = self.prefix + '/np'
 
     def set_np(self, np):
-        self.etcd.put(self.np_path, six.b('{}'.format(np)))
+        self.etcd.put(self.np_path, '{}'.format(np).encode('latin-1'))
 
     def scale_np(self, np):
         if self.etcd.get(self.np_path)[0] != None:

python/paddle/distributed/fleet/elastic/manager.py

@@ -262,7 +262,7 @@ def lease_heartbeat():
                         logger.info(
                             f"[lease_heartbeat] register host={self.curr_host}")
                         self.etcd.put(self.host_path,
-                                      six.b(self.curr_host),
+                                      self.curr_host.encode('latin-1'),
                                       lease=host_lease)
                 except Exception as e:
                     logger.error(
@@ -276,11 +276,15 @@ def lease_heartbeat():
                                              daemon=True)
         keepalived_thread.start()
 
-        self.etcd.put(self.host_path, six.b(self.curr_host), lease=host_lease)
+        self.etcd.put(self.host_path,
+                      self.curr_host.encode('latin-1'),
+                      lease=host_lease)
 
         # endpoints handle DISTRIBUTED_TRAINER_ENDPOINTS and PADDLE_TRAINERS
-        self.etcd.put(self.endpoints_path,
-                      six.b('{}|{}'.format(self.dist_endpoints, self.trainers)))
+        self.etcd.put(
+            self.endpoints_path,
+            '{}|{}'.format(self.dist_endpoints,
+                           self.trainers).encode('latin-1'))
 
         def endpoints_call_back(event):
             if not self.dist_endpoints:
@@ -433,7 +437,7 @@ def _match(self, host_list: list = None):
 
     def _update_endpoint(self, endpoints, hosts):
         self.etcd.put(self.endpoints_path,
-                      six.b('{}|{}'.format(endpoints, hosts)))
+                      '{}|{}'.format(endpoints, hosts).encode('latin-1'))
 
     def _update_fault_tolrance(self):
         rank = int(os.getenv('PADDLE_TRAINER_ID', -1))

python/paddle/distributed/fleet/launch.py

@@ -59,7 +59,6 @@
 import tempfile
 import os
 import time
-import six
 import copy
 import pathlib
 from argparse import ArgumentParser, REMAINDER
@@ -78,7 +77,7 @@
 
 def _print_arguments(args):
     print("-----------  Configuration Arguments -----------")
-    for arg, value in sorted(six.iteritems(vars(args))):
+    for arg, value in sorted(vars(args).items()):
         print("%s: %s" % (arg, value))
     print("------------------------------------------------")
 

python/paddle/distributed/fleet/launch_utils.py

@@ -24,7 +24,6 @@
 from contextlib import closing
 import multiprocessing
 import socket
-import six
 import struct
 import json
 
@@ -835,9 +834,7 @@ def get_device_proc_info(args):
                 "gpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(gpus), args.nproc_per_node)
 
             n = int(len(gpus) / int(args.nproc_per_node))
-            devices_per_proc = [
-                gpus[i:i + n] for i in six.moves.range(0, len(gpus), n)
-            ]
+            devices_per_proc = [gpus[i:i + n] for i in range(0, len(gpus), n)]
         else:
             devices_per_proc = gpus
     elif device_mode == DeviceMode.ASCEND_NPU:
@@ -847,9 +844,7 @@ def get_device_proc_info(args):
                 "npus' number:{} mod args.nproc_per_node:{} must == 0".format(len(npus), args.nproc_per_node)
 
             n = int(len(npus) / int(args.nproc_per_node))
-            devices_per_proc = [
-                npus[i:i + n] for i in six.moves.range(0, len(npus), n)
-            ]
+            devices_per_proc = [npus[i:i + n] for i in range(0, len(npus), n)]
         else:
             devices_per_proc = npus
     elif device_mode == DeviceMode.XPU:
@@ -859,9 +854,7 @@ def get_device_proc_info(args):
                 "xpus' number:{} mod args.nproc_per_node:{} must == 0".format(len(xpus), args.nproc_per_node)
 
             n = int(len(xpus) / int(args.nproc_per_node))
-            devices_per_proc = [
-                xpus[i:i + n] for i in six.moves.range(0, len(xpus), n)
-            ]
+            devices_per_proc = [xpus[i:i + n] for i in range(0, len(xpus), n)]
         else:
             devices_per_proc = xpus
     elif device_mode == DeviceMode.MLU:
@@ -871,9 +864,7 @@ def get_device_proc_info(args):
                 "mlus' number:{} mod args.nproc_per_node:{} must == 0".format(len(mlus), args.nproc_per_node)
 
             n = int(len(mlus) / int(args.nproc_per_node))
-            devices_per_proc = [
-                mlus[i:i + n] for i in six.moves.range(0, len(mlus), n)
-            ]
+            devices_per_proc = [mlus[i:i + n] for i in range(0, len(mlus), n)]
         else:
             devices_per_proc = mlus
     elif device_mode == DeviceMode.CPU: