bpo-43510: Implement PEP 597 opt-in EncodingWarning.

Doc/c-api/init_config.rst

@@ -583,6 +583,13 @@ PyConfig
 
       Default: ``0``.
 
+   .. c:member:: int warn_default_encoding
+
+      If equals to 1, emit a ``EncodingWarning`` when ``TextIOWrapper``
+      used its default encoding. See :pep:`597` for detail.
+
+      .. versionadded:: 3.10
+
    .. c:member:: wchar_t* check_hash_pycs_mode
 
       Control the validation behavior of hash-based ``.pyc`` files:

Doc/library/exceptions.rst

@@ -688,6 +688,14 @@ The following exceptions are used as warning categories; see the
    Base class for warnings generated by user code.
 
 
+.. exception:: EncodingWarning
+
+   Base class for warnings about encodings when those warnings are intended for
+   other Python developers.
+
+   .. versionadded:: 3.10
+
+
 .. exception:: DeprecationWarning
 
    Base class for warnings about deprecated features when those warnings are

Doc/library/io.rst

@@ -143,6 +143,32 @@ High-level Module Interface
    .. versionadded:: 3.8
 
 
+.. function:: text_encoding(encoding, stacklevel=1)
+
+   This is a helper function for functions that use :func:`open` or
+   :class:`TextIOWrapper` and take ``encoding=None`` option.
+
+   This function returns *encoding* if it is not ``None`` and "locale" if
+   *encoding* is ``None``.
+
+   This function emits an :class:`EncodingWarning` if
+   ``sys.flags.warn_default_encoding`` is true. *stacklevel* specifies where
+   the warning is emit for. For example::
+
+      def read_text(path, encoding=None):
+          encoding = io.text_encoding(encoding)  # stacklevel=1
+          with open(path, encoding) as f:
+              return f.read()
+
+   In this example, an :class:`EncodingWarning` is emit for the caller of the
+   ``read_text()``. If *stacklevel* is greater than 1, more stack frames are
+   skipped.
+
+   See :envvar:`PYTHONWARNDEFAULTENCODING` and :pep:`597` for more information.
+
+   .. versionadded:: 3.10
+
+
 .. exception:: BlockingIOError
 
    This is a compatibility alias for the builtin :exc:`BlockingIOError`
@@ -880,6 +906,11 @@ Text I/O
    encoded with.  It defaults to
    :func:`locale.getpreferredencoding(False) <locale.getpreferredencoding>`.
 
+   If ``sys.flags.warn_default_encoding`` is true and the default encoding
+   is used, this function emits an :class:`EncodingWarning`. You can suppress
+   the warning by using ``encoding="locale"`` option.
+   See :envvar:`PYTHONWARNDEFAULTENCODING` and :pep:`597` for more information.
+
    *errors* is an optional string that specifies how encoding and decoding
    errors are to be handled.  Pass ``'strict'`` to raise a :exc:`ValueError`
    exception if there is an encoding error (the default of ``None`` has the same
@@ -930,6 +961,9 @@ Text I/O
       locale encoding using :func:`locale.setlocale`, use the current locale
       encoding instead of the user preferred encoding.
 
+   .. versionchanged:: 3.10
+      *encoding* option now supports ``"locale"`` dummy encoding name.
+
    :class:`TextIOWrapper` provides these data attributes and methods in
    addition to those from :class:`TextIOBase` and :class:`IOBase`:
 

Doc/using/cmdline.rst

@@ -453,6 +453,9 @@ Miscellaneous options
    * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel
      tree rooted at the given directory instead of to the code tree. See also
      :envvar:`PYTHONPYCACHEPREFIX`.
+   * ``-X warn_default_encoding`` issues a :class:`EncodingWarning` when
+     an ``encoding`` option is omitted and the default encoding is locale-specific.
+     See also :envvar:`PYTHONWARNDEFAULTENCODING`.
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -482,6 +485,9 @@ Miscellaneous options
 
       The ``-X showalloccount`` option has been removed.
 
+   .. versionadded:: 3.10
+      The ``-X warn_default_encoding`` option.
+
    .. deprecated-removed:: 3.9 3.10
       The ``-X oldparser`` option.
 
@@ -907,6 +913,28 @@ conflict.
 
    .. versionadded:: 3.7
 
+.. envvar:: PYTHONWARNDEFAULTENCODING
+
+   If this environment variable is set to a non-empty string, issue a
+   :class:`EncodingWarning` when an ``encoding`` option is omitted and
+   the default encoding is locale-specific.
+
+   This option can be used to find bugs caused by not passing
+   ``encoding="utf8"`` option. For example::
+
+      # This code may cause UnicodeDecodeError on Windows.
+      # encoding="utf8" or "b" mode must be used.
+      with open(path) as f:
+          data = json.load(f)
+
+   ``encoding="locale"`` option can be used to specify locale-specific
+   encoding explicitly since Python 3.10. Python won't issue a
+   :class:`EncodingWarning` for it.
+
+   See :pep:`597` for detail.
+
+   .. versionadded:: 3.10
+
 
 Debug-mode variables
 ~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -153,6 +153,7 @@ typedef struct PyConfig {
     PyWideStringList warnoptions;
     int site_import;
     int bytes_warning;
+    int warn_default_encoding;
     int inspect;
     int interactive;
     int optimization_level;

Include/internal/pycore_initconfig.h

@@ -102,13 +102,15 @@ typedef struct {
     int isolated;             /* -I option */
     int use_environment;      /* -E option */
     int dev_mode;             /* -X dev and PYTHONDEVMODE */
+    int warn_default_encoding;     /* -X warn_default_encoding and PYTHONWARNDEFAULTENCODING */
 } _PyPreCmdline;
 
 #define _PyPreCmdline_INIT \
     (_PyPreCmdline){ \
         .use_environment = -1, \
         .isolated = -1, \
-        .dev_mode = -1}
+        .dev_mode = -1, \
+        .warn_default_encoding = -1}
 /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */
 
 extern void _PyPreCmdline_Clear(_PyPreCmdline *cmdline);

Include/pyerrors.h

@@ -146,6 +146,7 @@ PyAPI_DATA(PyObject *) PyExc_FutureWarning;
 PyAPI_DATA(PyObject *) PyExc_ImportWarning;
 PyAPI_DATA(PyObject *) PyExc_UnicodeWarning;
 PyAPI_DATA(PyObject *) PyExc_BytesWarning;
+PyAPI_DATA(PyObject *) PyExc_EncodingWarning;
 PyAPI_DATA(PyObject *) PyExc_ResourceWarning;
 
 

Lib/_pyio.py

@@ -40,6 +40,28 @@
 _CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
 
 
+def text_encoding(encoding, stacklevel=1):
+    """
+    Helper function to choose the text encoding.
+
+    When encoding is not None, just return it.
+    Otherwise, return the default text encoding (i.e. "locale").
+
+    This function emits EncodingWarning if *encoding* is None and
+    sys.flags.warn_default_encoding is true.
+
+    This function can be used in APIs having encoding=None option.
+    But please consider encoding="utf-8" for new APIs.
+    """
+    if encoding is None:
+        if sys.flags.warn_default_encoding:
+            import warnings
+            warnings.warn("'encoding' option is not specified.",
+                          EncodingWarning, stacklevel + 2)
+        encoding = "locale"
+    return encoding
+
+
 def open(file, mode="r", buffering=-1, encoding=None, errors=None,
          newline=None, closefd=True, opener=None):
 
@@ -248,6 +270,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
         result = buffer
         if binary:
             return result
+        encoding = text_encoding(encoding)
         text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering)
         result = text
         text.mode = mode
@@ -2004,19 +2027,22 @@ class TextIOWrapper(TextIOBase):
     def __init__(self, buffer, encoding=None, errors=None, newline=None,
                  line_buffering=False, write_through=False):
         self._check_newline(newline)
-        if encoding is None:
+        encoding = text_encoding(encoding)
+
+        if encoding == "locale":
             try:
-                encoding = os.device_encoding(buffer.fileno())
+                encoding = os.device_encoding(buffer.fileno()) or "locale"
             except (AttributeError, UnsupportedOperation):
                 pass
-            if encoding is None:
-                try:
-                    import locale
-                except ImportError:
-                    # Importing locale may fail if Python is being built
-                    encoding = "ascii"
-                else:
-                    encoding = locale.getpreferredencoding(False)
+
+        if encoding == "locale":
+            try:
+                import locale
+            except ImportError:
+                # Importing locale may fail if Python is being built
+                encoding = "utf-8"
+            else:
+                encoding = locale.getpreferredencoding(False)
 
         if not isinstance(encoding, str):
             raise ValueError("invalid encoding: %r" % encoding)

Lib/bz2.py

@@ -311,6 +311,7 @@ def open(filename, mode="rb", compresslevel=9,
     binary_file = BZ2File(filename, bz_mode, compresslevel=compresslevel)
 
     if "t" in mode:
+        encoding = io.text_encoding(encoding)
         return io.TextIOWrapper(binary_file, encoding, errors, newline)
     else:
         return binary_file

Lib/configparser.py

@@ -690,6 +690,7 @@ def read(self, filenames, encoding=None):
         """
         if isinstance(filenames, (str, bytes, os.PathLike)):
             filenames = [filenames]
+        encoding = io.text_encoding(encoding)
         read_ok = []
         for filename in filenames:
             try:

Lib/gzip.py

@@ -62,6 +62,7 @@ def open(filename, mode="rb", compresslevel=_COMPRESS_LEVEL_BEST,
         raise TypeError("filename must be a str or bytes object, or a file")
 
     if "t" in mode:
+        encoding = io.text_encoding(encoding)
         return io.TextIOWrapper(binary_file, encoding, errors, newline)
     else:
         return binary_file

Lib/io.py

@@ -54,7 +54,7 @@
 from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation,
                  open, open_code, FileIO, BytesIO, StringIO, BufferedReader,
                  BufferedWriter, BufferedRWPair, BufferedRandom,
-                 IncrementalNewlineDecoder, TextIOWrapper)
+                 IncrementalNewlineDecoder, text_encoding, TextIOWrapper)
 
 OpenWrapper = _io.open # for compatibility with _pyio
 

Lib/lzma.py

@@ -302,6 +302,7 @@ def open(filename, mode="rb", *,
                            preset=preset, filters=filters)
 
     if "t" in mode:
+        encoding = io.text_encoding(encoding)
         return io.TextIOWrapper(binary_file, encoding, errors, newline)
     else:
         return binary_file

Lib/pathlib.py

@@ -1241,6 +1241,8 @@ def open(self, mode='r', buffering=-1, encoding=None,
         Open the file pointed by this path and return a file object, as
         the built-in open() function does.
         """
+        if "b" not in mode:
+            encoding = io.text_encoding(encoding)
         return io.open(self, mode, buffering, encoding, errors, newline,
                        opener=self._opener)
 
@@ -1255,6 +1257,7 @@ def read_text(self, encoding=None, errors=None):
         """
         Open the file in text mode, read it, and close the file.
         """
+        encoding = io.text_encoding(encoding)
         with self.open(mode='r', encoding=encoding, errors=errors) as f:
             return f.read()
 
@@ -1274,6 +1277,7 @@ def write_text(self, data, encoding=None, errors=None, newline=None):
         if not isinstance(data, str):
             raise TypeError('data must be str, not %s' %
                             data.__class__.__name__)
+        encoding = io.text_encoding(encoding)
         with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f:
             return f.write(data)
 

Lib/site.py

@@ -170,7 +170,9 @@ def addpackage(sitedir, name, known_paths):
     fullname = os.path.join(sitedir, name)
     _trace(f"Processing .pth file: {fullname!r}")
     try:
-        f = io.TextIOWrapper(io.open_code(fullname))
+        # locale encoding is not ideal especially on Windows. But we have used
+        # it for a long time. setuptools uses the locale encoding too.
+        f = io.TextIOWrapper(io.open_code(fullname), encoding="locale")
     except OSError:
         return
     with f:

Lib/tempfile.py

@@ -543,6 +543,9 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None,
     if _os.name == 'nt' and delete:
         flags |= _os.O_TEMPORARY
 
+    if "b" not in mode:
+        encoding = _io.text_encoding(encoding)
+
     (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type)
     try:
         file = _io.open(fd, mode, buffering=buffering,
@@ -583,6 +586,9 @@ def TemporaryFile(mode='w+b', buffering=-1, encoding=None,
         """
         global _O_TMPFILE_WORKS
 
+        if "b" not in mode:
+            encoding = _io.text_encoding(encoding)
+
         prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir)
 
         flags = _bin_openflags
@@ -638,6 +644,7 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1,
         if 'b' in mode:
             self._file = _io.BytesIO()
         else:
+            encoding = _io.text_encoding(encoding)
             self._file = _io.TextIOWrapper(_io.BytesIO(),
                             encoding=encoding, errors=errors,
                             newline=newline)

Lib/test/exception_hierarchy.txt

@@ -61,4 +61,5 @@ BaseException
            +-- ImportWarning
            +-- UnicodeWarning
            +-- BytesWarning
+           +-- EncodingWarning
            +-- ResourceWarning

Lib/test/test_embed.py

@@ -389,6 +389,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         'site_import': 1,
         'bytes_warning': 0,
+        'warn_default_encoding': 0,
         'inspect': 0,
         'interactive': 0,
         'optimization_level': 0,

Lib/test/test_io.py

@@ -4249,6 +4249,29 @@ def test_check_encoding_errors(self):
         proc = assert_python_failure('-X', 'dev', '-c', code)
         self.assertEqual(proc.rc, 10, proc)
 
+    def test_check_encoding_warning(self):
+        # PEP 597: Raise warning when encoding is not specified
+        # and dev mode is enabled.
+        mod = self.io.__name__
+        filename = __file__
+        code = textwrap.dedent(f'''\
+            import sys
+            from {mod} import open, TextIOWrapper
+            import pathlib
+
+            with open({filename!r}) as f:           # line 5
+                pass
+
+            pathlib.Path({filename!r}).read_text()  # line 8
+        ''')
+        proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code)
+        warnings = proc.err.splitlines()
+        self.assertEqual(len(warnings), 2)
+        self.assertTrue(
+            warnings[0].startswith(b"<string>:5: EncodingWarning: "))
+        self.assertTrue(
+            warnings[1].startswith(b"<string>:8: EncodingWarning: "))
+
 
 class CMiscIOTest(MiscIOTest):
     io = io

Lib/test/test_pickle.py

@@ -483,7 +483,8 @@ def test_exceptions(self):
                 if exc in (BlockingIOError,
                            ResourceWarning,
                            StopAsyncIteration,
-                           RecursionError):
+                           RecursionError,
+                           EncodingWarning):
                     continue
                 if exc is not OSError and issubclass(exc, OSError):
                     self.assertEqual(reverse_mapping('builtins', name),