diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 21a375361bf58e7..395516f29b037c5 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -39,6 +39,7 @@ body: - "3.10" - "3.11" - "3.12" + - "3.13" - "CPython main branch" validations: required: true diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 5ab9f1cab23ef84..5fa37963e07eff7 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -971,8 +971,8 @@ These are the UTF-8 codec APIs: returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. + On error, set an exception, set *size* to ``-1`` (if it's not NULL) and + return ``NULL``. This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not @@ -992,11 +992,19 @@ These are the UTF-8 codec APIs: As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. + Raise an exception if the *unicode* string contains embedded null + characters. To accept embedded null characters and truncate on purpose + at the first null byte, ``PyUnicode_AsUTF8AndSize(unicode, NULL)`` can be + used instead. + .. versionadded:: 3.3 .. versionchanged:: 3.7 The return type is now ``const char *`` rather of ``char *``. + .. versionchanged:: 3.13 + Raise an exception if the string contains embedded null characters. + UTF-32 Codecs """"""""""""" diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 811b1bd84d24174..52d6d967d663270 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -726,6 +726,7 @@ function,PyUnicode_AsUCS4,3.7,, function,PyUnicode_AsUCS4Copy,3.7,, function,PyUnicode_AsUTF16String,3.2,, function,PyUnicode_AsUTF32String,3.2,, +function,PyUnicode_AsUTF8,3.13,, function,PyUnicode_AsUTF8AndSize,3.10,, function,PyUnicode_AsUTF8String,3.2,, function,PyUnicode_AsUnicodeEscapeString,3.2,, diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4cd05b0cf019de..2d5412d6b43e8bc 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -239,7 +239,7 @@ Glossary context manager An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`__enter__` and :meth:`__exit__` methods. + statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. See :pep:`343`. context variable @@ -636,7 +636,7 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`__getitem__` method + with an :meth:`__iter__` method or with a :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. Iterables can be @@ -1078,17 +1078,17 @@ Glossary sequence An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`__getitem__` special method and defines a + indices via the :meth:`~object.__getitem__` special method and defines a :meth:`__len__` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`__getitem__` and :meth:`__len__`, but is considered a + supports :meth:`~object.__getitem__` and :meth:`__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary :term:`immutable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, + :meth:`~object.__getitem__` and :meth:`__len__`, adding :meth:`count`, :meth:`index`, :meth:`__contains__`, and :meth:`__reversed__`. Types that implement this expanded interface can be registered explicitly using diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 274b2d69f0ab5c5..fb4f9da169c5abf 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -154,7 +154,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, even though it does not define an :meth:`~iterator.__iter__` method (it uses the old-style iterable protocol, defined in terms of :meth:`__len__` and - :meth:`__getitem__`). Note that this will not make ``get_iterator`` + :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` available as a method of ``Foo``, so it is provided separately. diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 3d7f43c86a0557d..091840781bd235b 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -23,20 +23,25 @@ build applications which provide an interactive interpreter prompt. ``'__doc__'`` set to ``None``. -.. class:: InteractiveConsole(locals=None, filename="") +.. class:: InteractiveConsole(locals=None, filename="", local_exit=False) Closely emulate the behavior of the interactive Python interpreter. This class builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. + ``sys.ps1`` and ``sys.ps2``, and input buffering. If *local_exit* is True, + ``exit()`` and ``quit()`` in the console will not raise :exc:`SystemExit`, but + instead return to the calling code. + .. versionchanged:: 3.13 + Added *local_exit* parameter. -.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None) +.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False) Convenience function to run a read-eval-print loop. This creates a new instance of :class:`InteractiveConsole` and sets *readfunc* to be used as the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` + use as the default namespace for the interpreter loop. If *local_exit* is provided, + it is passed to the :class:`InteractiveConsole` constructor. The :meth:`interact` method of the instance is then run with *banner* and *exitmsg* passed as the banner and exit message to use, if provided. The console object is discarded after use. @@ -44,6 +49,8 @@ build applications which provide an interactive interpreter prompt. .. versionchanged:: 3.6 Added *exitmsg* parameter. + .. versionchanged:: 3.13 + Added *local_exit* parameter. .. function:: compile_command(source, filename="", symbol="single") diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 2db4a67d1973d5b..4617624686b1f3b 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -520,44 +520,46 @@ The base :class:`Codec` class defines these methods which also define the function interfaces of the stateless encoder and decoder: -.. method:: Codec.encode(input, errors='strict') +.. class:: Codec - Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular - character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). + .. method:: encode(input, errors='strict') - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + Encodes the object *input* and returns a tuple (output object, length consumed). + For instance, :term:`text encoding` converts + a string object to a bytes object using a particular + character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The encoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamWriter` for codecs which have to keep state in order to make + encoding efficient. + The encoder must be able to handle zero length input and return an empty object + of the output object type in this situation. -.. method:: Codec.decode(input, errors='strict') - Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular - character set encoding to a string object. + .. method:: decode(input, errors='strict') - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only - buffer interface -- for example, buffer objects and memory mapped files. + Decodes the object *input* and returns a tuple (output object, length + consumed). For instance, for a :term:`text encoding`, decoding converts + a bytes object encoded using a particular + character set encoding to a string object. - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. + For text encodings and bytes-to-bytes codecs, + *input* must be a bytes object or one which provides the read-only + buffer interface -- for example, buffer objects and memory mapped files. - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. + The *errors* argument defines the error handling to apply. + It defaults to ``'strict'`` handling. - The decoder must be able to handle zero length input and return an empty object - of the output object type in this situation. + The method may not store state in the :class:`Codec` instance. Use + :class:`StreamReader` for codecs which have to keep state in order to make + decoding efficient. + + The decoder must be able to handle zero length input and return an empty object + of the output object type in this situation. Incremental Encoding and Decoding @@ -705,7 +707,7 @@ Stream Encoding and Decoding The :class:`StreamWriter` and :class:`StreamReader` classes provide generic working interfaces which can be used to implement new encoding submodules very -easily. See :mod:`encodings.utf_8` for an example of how this is done. +easily. See :mod:`!encodings.utf_8` for an example of how this is done. .. _stream-writer-objects: @@ -895,9 +897,10 @@ The design is such that one can use the factory functions returned by the .. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`read` and :meth:`write`, while *Reader* and *Writer* - work on the backend — the data in *stream*. + *encode* and *decode* work on the frontend — the data visible to + code calling :meth:`~StreamReader.read` and :meth:`~StreamWriter.write`, + while *Reader* and *Writer* + work on the backend — the data in *stream*. You can use these objects to do transparent transcodings, e.g., from Latin-1 to UTF-8 and back. @@ -1417,8 +1420,8 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` | | quotedprintable, | quoted printable. | ``quotetabs=True`` / | | | quoted_printable | | :meth:`quopri.decode` | +----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`uu.encode` / | -| | | uuencode. | :meth:`uu.decode` | +| uu_codec | uu | Convert the operand using | | +| | | uuencode. | | +----------------------+------------------+------------------------------+------------------------------+ | zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | | | | gzip. | :meth:`zlib.decompress` | diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 43a3286ba832cf0..edc078953290d7a 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -192,7 +192,7 @@ ABC Inherits from Abstract Methods Mi .. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are registered as :class:`Iterable` or that have an :meth:`__iter__` method, but it does not detect classes that iterate with the - :meth:`__getitem__` method. The only reliable way to determine + :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -222,7 +222,7 @@ Collections Abstract Base Classes -- Detailed Descriptions Checking ``isinstance(obj, Iterable)`` detects classes that are registered as :class:`Iterable` or that have an :meth:`__iter__` method, but it does - not detect classes that iterate with the :meth:`__getitem__` method. + not detect classes that iterate with the :meth:`~object.__getitem__` method. The only reliable way to determine whether an object is :term:`iterable` is to call ``iter(obj)``. @@ -262,8 +262,8 @@ Collections Abstract Base Classes -- Detailed Descriptions Implementation note: Some of the mixin methods, such as :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`__getitem__` method. - Consequently, if :meth:`__getitem__` is implemented with constant + repeated calls to the underlying :meth:`~object.__getitem__` method. + Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 02df6e9f1379264..17dd6da7479e504 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -743,12 +743,12 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, If calling :attr:`default_factory` raises an exception this exception is propagated unchanged. - This method is called by the :meth:`__getitem__` method of the + This method is called by the :meth:`~object.__getitem__` method of the :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. + returns or raises is then returned or raised by :meth:`~object.__getitem__`. Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal + :meth:`~object.__getitem__`. This means that :meth:`get` will, like normal dictionaries, return ``None`` as a default rather than using :attr:`default_factory`. diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 7cd081d1f54f439..66b9c1371052235 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -45,7 +45,7 @@ Functions and classes provided: This function is a :term:`decorator` that can be used to define a factory function for :keyword:`with` statement context managers, without needing to - create a class or separate :meth:`__enter__` and :meth:`__exit__` methods. + create a class or separate :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. While many objects natively support use in with statements, sometimes a resource needs to be managed that isn't a context manager in its own right, @@ -515,7 +515,7 @@ Functions and classes provided: # the with statement, even if attempts to open files later # in the list raise an exception - The :meth:`__enter__` method returns the :class:`ExitStack` instance, and + The :meth:`~object.__enter__` method returns the :class:`ExitStack` instance, and performs no additional operations. Each instance maintains a stack of registered callbacks that are called in @@ -543,9 +543,9 @@ Functions and classes provided: .. method:: enter_context(cm) - Enters a new context manager and adds its :meth:`__exit__` method to + Enters a new context manager and adds its :meth:`~object.__exit__` method to the callback stack. The return value is the result of the context - manager's own :meth:`__enter__` method. + manager's own :meth:`~object.__enter__` method. These context managers may suppress exceptions just as they normally would if used directly as part of a :keyword:`with` statement. @@ -556,18 +556,18 @@ Functions and classes provided: .. method:: push(exit) - Adds a context manager's :meth:`__exit__` method to the callback stack. + Adds a context manager's :meth:`~object.__exit__` method to the callback stack. As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`__enter__` implementation with a context manager's own - :meth:`__exit__` method. + part of an :meth:`~object.__enter__` implementation with a context manager's own + :meth:`~object.__exit__` method. If passed an object that is not a context manager, this method assumes it is a callback with the same signature as a context manager's - :meth:`__exit__` method and adds it directly to the callback stack. + :meth:`~object.__exit__` method and adds it directly to the callback stack. By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`__exit__` methods can. + same way context manager :meth:`~object.__exit__` methods can. The passed in object is returned from the function, allowing this method to be used as a function decorator. @@ -714,7 +714,7 @@ Cleaning up in an ``__enter__`` implementation As noted in the documentation of :meth:`ExitStack.push`, this method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`__enter__` implementation fail. +steps in the :meth:`~object.__enter__` implementation fail. Here's an example of doing this for a context manager that accepts resource acquisition and release functions, along with an optional validation function, @@ -871,7 +871,7 @@ And also as a function decorator:: Note that there is one additional limitation when using context managers as function decorators: there's no way to access the return value of -:meth:`__enter__`. If that value is needed, then it is still necessary to use +:meth:`~object.__enter__`. If that value is needed, then it is still necessary to use an explicit ``with`` statement. .. seealso:: diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 54a8e79a0ef9417..ad013944ce3ca36 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -277,13 +277,34 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. -In addition, if ``M.__test__`` exists and "is true", it must be a dict, and each +In addition, there are cases when you want tests to be part of a module but not part +of the help text, which requires that the tests not be included in the docstring. +Doctest looks for a module-level variable called ``__test__`` and uses it to locate other +tests. If ``M.__test__`` exists and is truthy, it must be a dict, and each entry maps a (string) name to a function object, class object, or string. Function and class object docstrings found from ``M.__test__`` are searched, and strings are treated as if they were docstrings. In output, a key ``K`` in -``M.__test__`` appears with name :: +``M.__test__`` appears with name ``M.__test__.K``. - .__test__.K +For example, place this block of code at the top of :file:`example.py`: + +.. code-block:: python + + __test__ = { + 'numbers': """ + >>> factorial(6) + 720 + + >>> [factorial(n) for n in range(6)] + [1, 1, 2, 6, 24, 120] + """ + } + +The value of ``example.__test__["numbers"]`` will be treated as a +docstring and all the tests inside it will be run. It is +important to note that the value can be mapped to a function, +class object, or module; if so, :mod:`!doctest` +searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 5bef155a4af3103..c4c322a82e1f44d 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -367,7 +367,7 @@ Here are the methods of the :class:`Message` class: .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (defaults to ``None``). Here are some additional useful methods: diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 225f498781fa863..f58d93da6ed6871 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -247,7 +247,7 @@ message objects. .. method:: get(name, failobj=None) Return the value of the named header field. This is identical to - :meth:`__getitem__` except that optional *failobj* is returned if the + :meth:`~object.__getitem__` except that optional *failobj* is returned if the named header is missing (*failobj* defaults to ``None``). diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 6031ca854343621..a5f580c07bdbf12 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -983,7 +983,7 @@ are always available. They are listed here in alphabetical order. differently depending on the presence of the second argument. Without a second argument, *object* must be a collection object which supports the :term:`iterable` protocol (the :meth:`__iter__` method), or it must support - the sequence protocol (the :meth:`__getitem__` method with integer arguments + the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, then *object* must be a callable object. The iterator created in this case @@ -1563,7 +1563,7 @@ are always available. They are listed here in alphabetical order. Return a reverse :term:`iterator`. *seq* must be an object which has a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`__getitem__` method with integer + :meth:`__len__` method and the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index d5bbe67fb30a621..5c79daf0f47d8e6 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -43,7 +43,7 @@ The :mod:`getpass` module provides two functions: Return the "login name" of the user. This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`LNAME` and :envvar:`USERNAME`, in order, and + :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and returns the value of the first one which is set to a non-empty string. If none are set, the login name from the password database is returned on systems which support the :mod:`pwd` module, otherwise, an exception is diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 3efd3be59092546..b463c0b6d0e4020 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -640,6 +640,9 @@ function. Accepts a wide range of Python callables, from plain functions and classes to :func:`functools.partial` objects. + If the passed object has a ``__signature__`` attribute, this function + returns it without further computations. + For objects defined in modules using stringized annotations (``from __future__ import annotations``), :func:`signature` will attempt to automatically un-stringize the annotations using @@ -763,6 +766,8 @@ function. sig = MySignature.from_callable(min) assert isinstance(sig, MySignature) + Its behavior is otherwise identical to that of :func:`signature`. + .. versionadded:: 3.5 .. versionadded:: 3.10 diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 91df07d914cae29..b27deb20f13236a 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -167,7 +167,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. Return a representation of the message corresponding to *key*. If no such message exists, *default* is returned if the method was called as :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`__getitem__`. The message is represented as an instance + called as :meth:`~object.__getitem__`. The message is represented as an instance of the appropriate format-specific :class:`Message` subclass unless a custom message factory was specified when the :class:`Mailbox` instance was initialized. diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 57c67bcf3aa12e1..96f2c287875d412 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -306,7 +306,7 @@ expect a function argument. itemgetter(*items) Return a callable object that fetches *item* from its operand using the - operand's :meth:`__getitem__` method. If multiple items are specified, + operand's :meth:`~object.__getitem__` method. If multiple items are specified, returns a tuple of lookup values. For example: * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. @@ -326,7 +326,7 @@ expect a function argument. return tuple(obj[item] for item in items) return g - The items can be any type accepted by the operand's :meth:`__getitem__` + The items can be any type accepted by the operand's :meth:`~object.__getitem__` method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and strings accept an index or a slice: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4c63d1557c048ad..8160740c7a05bd1 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2267,7 +2267,7 @@ expression support in the :mod:`re` module). Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements - indexing via :meth:`__getitem__`, typically a :term:`mapping` or + indexing via :meth:`~object.__getitem__`, typically a :term:`mapping` or :term:`sequence`. When indexed by a Unicode ordinal (an integer), the table object can do any of the following: return a Unicode ordinal or a string, to map the character to one or more other characters; return @@ -4857,7 +4857,7 @@ before the statement body is executed and exited when the statement ends: The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`__exit__` + context management code to easily detect whether or not an :meth:`~object.__exit__` method has actually failed. Python defines several context managers to support easy thread synchronisation, diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 5dcbdaf8e5d0e44..1024f66f3264baf 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -1,14 +1,16 @@ :mod:`sys.monitoring` --- Execution event monitoring ==================================================== -.. module:: sys.monitoring - :synopsis: Access and control event monitoring +.. module:: sys.monitoring + :synopsis: Access and control event monitoring + +.. versionadded:: 3.12 ----------------- .. note:: - ``sys.monitoring`` is a namespace within the ``sys`` module, + :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, not an independent module, so there is no need to ``import sys.monitoring``, simply ``import sys`` and then use ``sys.monitoring``. @@ -18,45 +20,45 @@ This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`!sys.monitoring` namespace provides means to +monitor execution. The :mod:`sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: -* Tool identifiers -* Events -* Callbacks +* `Tool identifiers`_ +* `Events`_ +* :ref:`Callbacks ` Tool identifiers ---------------- -A tool identifier is an integer and associated name. +A tool identifier is an integer and the associated name. Tool identifiers are used to discourage tools from interfering with each other and to allow multiple tools to operate at the same time. Currently tools are completely independent and cannot be used to monitor each other. This restriction may be lifted in the future. Before registering or activating events, a tool should choose an identifier. -Identifiers are integers in the range 0 to 5. +Identifiers are integers in the range 0 to 5 inclusive. Registering and using tools ''''''''''''''''''''''''''' .. function:: use_tool_id(id: int, name: str) -> None - Must be called before ``id`` can be used. - ``id`` must be in the range 0 to 5 inclusive. - Raises a ``ValueError`` if ``id`` is in use. + Must be called before *id* can be used. + *id* must be in the range 0 to 5 inclusive. + Raises a :exc:`ValueError` if *id* is in use. .. function:: free_tool_id(id: int) -> None - Should be called once a tool no longer requires ``id``. + Should be called once a tool no longer requires *id*. .. function:: get_tool(id: int) -> str | None - Returns the name of the tool if ``id`` is in use, + Returns the name of the tool if *id* is in use, otherwise it returns ``None``. - ``id`` must be in the range 0 to 5 inclusive. + *id* must be in the range 0 to 5 inclusive. All IDs are treated the same by the VM with regard to events, but the following IDs are pre-defined to make co-operation of tools easier:: @@ -75,48 +77,89 @@ Events The following events are supported: -BRANCH - A conditional branch is taken (or not). -CALL - A call in Python code (event occurs before the call). -C_RAISE - Exception raised from any callable, except Python functions (event occurs after the exit). -C_RETURN - Return from any callable, except Python functions (event occurs after the return). -EXCEPTION_HANDLED - An exception is handled. -INSTRUCTION - A VM instruction is about to be executed. -JUMP - An unconditional jump in the control flow graph is made. -LINE - An instruction is about to be executed that has a different line number from the preceding instruction. -PY_RESUME - Resumption of a Python function (for generator and coroutine functions), except for throw() calls. -PY_RETURN - Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). -PY_START - Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) -PY_THROW - A Python function is resumed by a throw() call. -PY_UNWIND - Exit from a Python function during exception unwinding. -PY_YIELD - Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). -RAISE - An exception is raised, except those that cause a ``STOP_ITERATION`` event. -RERAISE - An exception is re-raised, for example at the end of a ``finally`` block. -STOP_ITERATION - An artificial ``StopIteration`` is raised; see `the STOP_ITERATION event`_. +.. monitoring-event:: BRANCH + + A conditional branch is taken (or not). + +.. monitoring-event:: CALL + + A call in Python code (event occurs before the call). + +.. monitoring-event:: C_RAISE + + An exception raised from any callable, except for Python functions (event occurs after the exit). + +.. monitoring-event:: C_RETURN + + Return from any callable, except for Python functions (event occurs after the return). + +.. monitoring-event:: EXCEPTION_HANDLED + + An exception is handled. + +.. monitoring-event:: INSTRUCTION + + A VM instruction is about to be executed. + +.. monitoring-event:: JUMP + + An unconditional jump in the control flow graph is made. + +.. monitoring-event:: LINE + + An instruction is about to be executed that has a different line number from the preceding instruction. + +.. monitoring-event:: PY_RESUME + + Resumption of a Python function (for generator and coroutine functions), except for ``throw()`` calls. + +.. monitoring-event:: PY_RETURN + + Return from a Python function (occurs immediately before the return, the callee's frame will be on the stack). + +.. monitoring-event:: PY_START + + Start of a Python function (occurs immediately after the call, the callee's frame will be on the stack) + +.. monitoring-event:: PY_THROW + + A Python function is resumed by a ``throw()`` call. + +.. monitoring-event:: PY_UNWIND + + Exit from a Python function during exception unwinding. + +.. monitoring-event:: PY_YIELD + + Yield from a Python function (occurs immediately before the yield, the callee's frame will be on the stack). + +.. monitoring-event:: RAISE + + An exception is raised, except those that cause a :monitoring-event:`STOP_ITERATION` event. + +.. monitoring-event:: RERAISE + + An exception is re-raised, for example at the end of a :keyword:`finally` block. + +.. monitoring-event:: STOP_ITERATION + + An artificial :exc:`StopIteration` is raised; see `the STOP_ITERATION event`_. + More events may be added in the future. These events are attributes of the :mod:`!sys.monitoring.events` namespace. Each event is represented as a power-of-2 integer constant. To define a set of events, simply bitwise or the individual events together. -For example, to specify both ``PY_RETURN`` and ``PY_START`` events, use the -expression ``PY_RETURN | PY_START``. +For example, to specify both :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_START` +events, use the expression ``PY_RETURN | PY_START``. + +.. monitoring-event:: NO_EVENTS + + An alias for ``0`` so users can do explict comparisions like:: + + if get_events(DEBUGGER_ID) == NO_EVENTS: + ... Events are divided into three groups: @@ -127,16 +170,16 @@ Local events are associated with normal execution of the program and happen at clearly defined locations. All local events can be disabled. The local events are: -* PY_START -* PY_RESUME -* PY_RETURN -* PY_YIELD -* CALL -* LINE -* INSTRUCTION -* JUMP -* BRANCH -* STOP_ITERATION +* :monitoring-event:`PY_START` +* :monitoring-event:`PY_RESUME` +* :monitoring-event:`PY_RETURN` +* :monitoring-event:`PY_YIELD` +* :monitoring-event:`CALL` +* :monitoring-event:`LINE` +* :monitoring-event:`INSTRUCTION` +* :monitoring-event:`JUMP` +* :monitoring-event:`BRANCH` +* :monitoring-event:`STOP_ITERATION` Ancillary events '''''''''''''''' @@ -144,12 +187,13 @@ Ancillary events Ancillary events can be monitored like other events, but are controlled by another event: -* C_RAISE -* C_RETURN +* :monitoring-event:`C_RAISE` +* :monitoring-event:`C_RETURN` -The ``C_RETURN`` and ``C_RAISE`` events are controlled by the ``CALL`` -event. ``C_RETURN`` and ``C_RAISE`` events will only be seen if the -corresponding ``CALL`` event is being monitored. +The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events +are controlled by the :monitoring-event:`CALL` event. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the +corresponding :monitoring-event:`CALL` event is being monitored. Other events '''''''''''' @@ -159,30 +203,31 @@ program and cannot be individually disabled. The other events that can be monitored are: -* PY_THROW -* PY_UNWIND -* RAISE -* EXCEPTION_HANDLED +* :monitoring-event:`PY_THROW` +* :monitoring-event:`PY_UNWIND` +* :monitoring-event:`RAISE` +* :monitoring-event:`EXCEPTION_HANDLED` The STOP_ITERATION event '''''''''''''''''''''''' :pep:`PEP 380 <380#use-of-stopiteration-to-return-values>` -specifies that a ``StopIteration`` exception is raised when returning a value +specifies that a :exc:`StopIteration` exception is raised when returning a value from a generator or coroutine. However, this is a very inefficient way to return a value, so some Python implementations, notably CPython 3.12+, do not raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators -and coroutines, the ``STOP_ITERATION`` event is provided. -``STOP_ITERATION`` can be locally disabled, unlike ``RAISE``. +and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. Turning events on and off ------------------------- -In order to monitor an event, it must be turned on and a callback registered. +In order to monitor an event, it must be turned on and a corresponding callback +must be registered. Events can be turned on or off by setting the events either globally or for a particular code object. @@ -198,8 +243,8 @@ Events can be controlled globally by modifying the set of events being monitored .. function:: set_events(tool_id: int, event_set: int) - Activates all events which are set in ``event_set``. - Raises a ``ValueError`` if ``tool_id`` is not in use. + Activates all events which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. No events are active by default. @@ -210,12 +255,12 @@ Events can also be controlled on a per code object basis. .. function:: get_local_events(tool_id: int, code: CodeType) -> int - Returns all the local events for ``code`` + Returns all the local events for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int) - Activates all the local events for ``code`` which are set in ``event_set``. - Raises a ``ValueError`` if ``tool_id`` is not in use. + Activates all the local events for *code* which are set in *event_set*. + Raises a :exc:`ValueError` if *tool_id* is not in use. Local events add to global events, but do not mask them. In other words, all global events will trigger for a code object, @@ -225,8 +270,13 @@ regardless of the local events. Disabling events '''''''''''''''' +.. data:: DISABLE + + A special value that can be returned from a callback function to disable + events for the current code location. + Local events can be disabled for a specific code location by returning -``sys.monitoring.DISABLE`` from a callback function. This does not change +:data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. Disabling events for specific locations is very important for high @@ -235,6 +285,8 @@ debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +.. _callbacks: + Registering callback functions ------------------------------ @@ -242,11 +294,11 @@ To register a callable for events call .. function:: register_callback(tool_id: int, event: int, func: Callable | None) -> Callable | None - Registers the callable ``func`` for the ``event`` with the given ``tool_id`` + Registers the callable *func* for the *event* with the given *tool_id* - If another callback was registered for the given ``tool_id`` and ``event``, + If another callback was registered for the given *tool_id* and *event*, it is unregistered and returned. - Otherwise ``register_callback`` returns ``None``. + Otherwise :func:`register_callback` returns ``None``. Functions can be unregistered by calling @@ -254,47 +306,51 @@ Functions can be unregistered by calling Callback functions can be registered and unregistered at any time. -Registering or unregistering a callback function will generate a ``sys.audit`` event. +Registering or unregistering a callback function will generate a :func:`sys.audit` event. Callback function arguments ''''''''''''''''''''''''''' +.. data:: MISSING + + A special value that is passed to a callback function to indicate + that there are no arguments to the call. + When an active event occurs, the registered callback function is called. Different events will provide the callback function with different arguments, as follows: -* ``PY_START`` and ``PY_RESUME``:: +* :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: func(code: CodeType, instruction_offset: int) -> DISABLE | Any -* ``PY_RETURN`` and ``PY_YIELD``: +* :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - ``func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any -* ``CALL``, ``C_RAISE`` and ``C_RETURN``: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: - ``func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any - If there are no arguments, ``arg0`` is set to ``MISSING``. + If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. -* ``RAISE``, ``RERAISE``, ``EXCEPTION_HANDLED``, ``PY_UNWIND``, ``PY_THROW`` and ``STOP_ITERATION``: +* :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, + :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - ``func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any -* ``LINE``: +* :monitoring-event:`LINE`:: - ``func(code: CodeType, line_number: int) -> DISABLE | Any`` + func(code: CodeType, line_number: int) -> DISABLE | Any -* ``BRANCH`` and ``JUMP``: +* :monitoring-event:`BRANCH` and :monitoring-event:`JUMP`:: - ``func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any`` + func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any - Note that the ``destination_offset`` is where the code will next execute. + Note that the *destination_offset* is where the code will next execute. For an untaken branch this will be the offset of the instruction following the branch. -* ``INSTRUCTION``: - - ``func(code: CodeType, instruction_offset: int) -> DISABLE | Any`` - +* :monitoring-event:`INSTRUCTION`:: + func(code: CodeType, instruction_offset: int) -> DISABLE | Any diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f9f556306f58275..8eeeefa0133d127 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1554,9 +1554,8 @@ always available. function to be used for the new scope, or ``None`` if the scope shouldn't be traced. - The local trace function should return a reference to itself (or to another - function for further tracing in that scope), or ``None`` to turn off tracing - in that scope. + The local trace function should return a reference to itself, or to another + function which would then be used as the local trace function for the scope. If there is any error occurred in the trace function, it will be unset, just like ``settrace(None)`` is called. diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index fb1ff567d49e5c8..03806178e9d3fbd 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -43,10 +43,20 @@ The module defines the following functions: Set the tty attributes for file descriptor *fd* from the *attributes*, which is a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: :const:`TCSANOW` to change - immediately, :const:`TCSADRAIN` to change after transmitting all queued output, - or :const:`TCSAFLUSH` to change after transmitting all queued output and - discarding all queued input. + determines when the attributes are changed: + + .. data:: TCSANOW + + Change attributes immediately. + + .. data:: TCSADRAIN + + Change attributes after transmitting all queued output. + + .. data:: TCSAFLUSH + + Change attributes after transmitting all queued output and + discarding all queued input. .. function:: tcsendbreak(fd, duration) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index de60151bb32ce12..6be77260faa7ea7 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1043,7 +1043,7 @@ The :mod:`test.support` module defines the following classes: :const:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file creation. - On both platforms, the old value is restored by :meth:`__exit__`. + On both platforms, the old value is restored by :meth:`~object.__exit__`. .. class:: SaveSignals() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index ea6412abf72d1a7..d6ac4d09300d9fe 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -1707,7 +1707,7 @@ Keywords can be used in the :func:`patch.dict` call to set values in the diction :func:`patch.dict` can be used with dictionary like objects that aren't actually dictionaries. At the very minimum they must support item getting, setting, deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either +magic methods :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either :meth:`__iter__` or :meth:`__contains__`. >>> class Container: @@ -2531,8 +2531,8 @@ are closed properly and is becoming common:: f.write('something') The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`__enter__` and -:meth:`__exit__` called). +*returned object* that is used as a context manager (and has :meth:`~object.__enter__` and +:meth:`~object.__exit__` called). Mocking context managers with a :class:`MagicMock` is common enough and fiddly enough that a helper function is useful. :: diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 39a4c1ba1423381..be9e56b04c1fbf1 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -180,7 +180,7 @@ also provides these miscellaneous utilities: print(chunk) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. :mod:`wsgiref.headers` -- WSGI response header tools @@ -201,7 +201,7 @@ manipulation of WSGI response headers using a mapping-like interface. an empty list. :class:`Headers` objects support typical mapping operations including - :meth:`__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, + :meth:`~object.__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, :meth:`__delitem__` and :meth:`__contains__`. For each of these methods, the key is the header name (treated case-insensitively), and the value is the first value associated with that header name. Setting a header diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index d1df465a598e532..843c2fd7fdb937a 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -115,7 +115,7 @@ DOMEventStream Objects .. class:: DOMEventStream(stream, parser, bufsize) .. versionchanged:: 3.11 - Support for :meth:`__getitem__` method has been removed. + Support for :meth:`~object.__getitem__` method has been removed. .. method:: getEvent() diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f79db828604b246..8f6481339837a05 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -489,37 +489,37 @@ The execution of the :keyword:`with` statement with one "item" proceeds as follo #. The context expression (the expression given in the :token:`~python-grammar:with_item`) is evaluated to obtain a context manager. -#. The context manager's :meth:`__enter__` is loaded for later use. +#. The context manager's :meth:`~object.__enter__` is loaded for later use. -#. The context manager's :meth:`__exit__` is loaded for later use. +#. The context manager's :meth:`~object.__exit__` is loaded for later use. -#. The context manager's :meth:`__enter__` method is invoked. +#. The context manager's :meth:`~object.__enter__` method is invoked. #. If a target was included in the :keyword:`with` statement, the return value - from :meth:`__enter__` is assigned to it. + from :meth:`~object.__enter__` is assigned to it. .. note:: - The :keyword:`with` statement guarantees that if the :meth:`__enter__` - method returns without an error, then :meth:`__exit__` will always be + The :keyword:`with` statement guarantees that if the :meth:`~object.__enter__` + method returns without an error, then :meth:`~object.__exit__` will always be called. Thus, if an error occurs during the assignment to the target list, it will be treated the same as an error occurring within the suite would be. See step 7 below. #. The suite is executed. -#. The context manager's :meth:`__exit__` method is invoked. If an exception +#. The context manager's :meth:`~object.__exit__` method is invoked. If an exception caused the suite to be exited, its type, value, and traceback are passed as - arguments to :meth:`__exit__`. Otherwise, three :const:`None` arguments are + arguments to :meth:`~object.__exit__`. Otherwise, three :const:`None` arguments are supplied. If the suite was exited due to an exception, and the return value from the - :meth:`__exit__` method was false, the exception is reraised. If the return + :meth:`~object.__exit__` method was false, the exception is reraised. If the return value was true, the exception is suppressed, and execution continues with the statement following the :keyword:`with` statement. If the suite was exited for any reason other than an exception, the return - value from :meth:`__exit__` is ignored, and execution proceeds at the normal + value from :meth:`~object.__exit__` is ignored, and execution proceeds at the normal location for the kind of exit that was taken. The following code:: @@ -1058,7 +1058,7 @@ subject value: .. note:: Key-value pairs are matched using the two-argument form of the mapping subject's ``get()`` method. Matched key-value pairs must already be present in the mapping, and not created on-the-fly via :meth:`__missing__` or - :meth:`__getitem__`. + :meth:`~object.__getitem__`. In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following happens: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index cc81f73f8243520..9e9fe831f4a647a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2939,7 +2939,7 @@ For more information on context managers, see :ref:`typecontextmanager`. (i.e., prevent it from being propagated), it should return a true value. Otherwise, the exception will be processed normally upon exit from this method. - Note that :meth:`__exit__` methods should not reraise the passed-in exception; + Note that :meth:`~object.__exit__` methods should not reraise the passed-in exception; this is the caller's responsibility. @@ -3257,12 +3257,12 @@ Asynchronous context managers can be used in an :keyword:`async with` statement. .. method:: object.__aenter__(self) - Semantically similar to :meth:`__enter__`, the only + Semantically similar to :meth:`~object.__enter__`, the only difference being that it must return an *awaitable*. .. method:: object.__aexit__(self, exc_type, exc_value, traceback) - Semantically similar to :meth:`__exit__`, the only + Semantically similar to :meth:`~object.__exit__`, the only difference being that it must return an *awaitable*. An example of an asynchronous context manager class:: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 5e9d12c0087a946..88d4fca4e3267cc 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -889,7 +889,7 @@ to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The resulting value must be a nonnegative integer less than the number of items in the sequence, and the subscription selects the item whose index is that value (counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`__getitem__` method, subclasses overriding +occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding this method will need to explicitly add that support. .. index:: @@ -944,7 +944,7 @@ slice list contains no proper slice). single: step (slice object attribute) The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`__getitem__` method as +same :meth:`~object.__getitem__` method as normal subscription) with a key that is constructed from the slice list, as follows. If the slice list contains at least one comma, the key is a tuple containing the conversion of the slice items; otherwise, the conversion of the @@ -1670,7 +1670,7 @@ If an exception is raised during the iteration, it is as if :keyword:`in` raised that exception. Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative +:meth:`~object.__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index raises the :exc:`IndexError` exception. (If any other exception is raised, it is as if :keyword:`in` raised that exception). diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f62ed55282b8d24..5b21d11271d598b 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -42,7 +42,6 @@ Doc/library/bisect.rst Doc/library/bz2.rst Doc/library/calendar.rst Doc/library/cmd.rst -Doc/library/codecs.rst Doc/library/collections.abc.rst Doc/library/collections.rst Doc/library/concurrent.futures.rst @@ -67,7 +66,6 @@ Doc/library/fcntl.rst Doc/library/ftplib.rst Doc/library/functions.rst Doc/library/functools.rst -Doc/library/getpass.rst Doc/library/gettext.rst Doc/library/gzip.rst Doc/library/http.client.rst @@ -121,7 +119,6 @@ Doc/library/tkinter.rst Doc/library/tkinter.scrolledtext.rst Doc/library/tkinter.ttk.rst Doc/library/traceback.rst -Doc/library/tty.rst Doc/library/unittest.mock.rst Doc/library/unittest.rst Doc/library/urllib.parse.rst diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 4ba5f5a1fbdc1b0..11d954adaecead7 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -607,6 +607,13 @@ def parse_pdb_command(env, sig, signode): return fullname +def parse_monitoring_event(env, sig, signode): + """Transform a monitoring event signature into RST nodes.""" + signode += addnodes.desc_addname('sys.monitoring.events.', 'sys.monitoring.events.') + signode += addnodes.desc_name(sig, sig) + return sig + + def process_audit_events(app, doctree, fromdocname): for node in doctree.traverse(audit_event_list): break @@ -707,6 +714,7 @@ def setup(app): app.add_builder(PydocTopicsBuilder) app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) + app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event) app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index d9ead57413cbbfa..6dfe79cef00987f 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -424,22 +424,22 @@ Another significant addition to 2.2 is an iteration interface at both the C and Python levels. Objects can define how they can be looped over by callers. In Python versions up to 2.1, the usual way to make ``for item in obj`` work is -to define a :meth:`__getitem__` method that looks something like this:: +to define a :meth:`~object.__getitem__` method that looks something like this:: def __getitem__(self, index): return -:meth:`__getitem__` is more properly used to define an indexing operation on an +:meth:`~object.__getitem__` is more properly used to define an indexing operation on an object so that you can write ``obj[5]`` to retrieve the sixth element. It's a bit misleading when you're using this only to support :keyword:`for` loops. Consider some file-like object that wants to be looped over; the *index* parameter is essentially meaningless, as the class probably assumes that a -series of :meth:`__getitem__` calls will be made with *index* incrementing by -one each time. In other words, the presence of the :meth:`__getitem__` method +series of :meth:`~object.__getitem__` calls will be made with *index* incrementing by +one each time. In other words, the presence of the :meth:`~object.__getitem__` method doesn't mean that using ``file[5]`` to randomly access the sixth element will work, though it really should. -In Python 2.2, iteration can be implemented separately, and :meth:`__getitem__` +In Python 2.2, iteration can be implemented separately, and :meth:`~object.__getitem__` methods can be limited to classes that really do support random access. The basic idea of iterators is simple. A new built-in function, ``iter(obj)`` or ``iter(C, sentinel)``, is used to get an iterator. ``iter(obj)`` returns diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index ec1ca417ee139d7..af332b28a28231e 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -925,7 +925,7 @@ Deletion is more straightforward:: >>> a [1, 3] -One can also now pass slice objects to the :meth:`__getitem__` methods of the +One can also now pass slice objects to the :meth:`~object.__getitem__` methods of the built-in sequences:: >>> range(10).__getitem__(slice(0, 5, 2)) @@ -1596,7 +1596,7 @@ complete list of changes, or look through the CVS logs for all the details. module. Adding the mix-in as a superclass provides the full dictionary interface - whenever the class defines :meth:`__getitem__`, :meth:`__setitem__`, + whenever the class defines :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and :meth:`keys`. For example:: >>> import UserDict diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 2df6d603207a10e..a1bdcc236e82b70 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -575,15 +575,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -609,7 +609,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -652,10 +652,10 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``'as VAR'`` clause is present, the value is simply discarded. @@ -669,7 +669,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -703,7 +703,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback (self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -715,7 +715,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -748,10 +748,10 @@ are useful for writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Our database example from the previous section could be written using this diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 198d5152194e931..016de153f3dd6ac 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -269,15 +269,15 @@ structure is:: with-block The expression is evaluated, and it should result in an object that supports the -context management protocol (that is, has :meth:`__enter__` and :meth:`__exit__` +context management protocol (that is, has :meth:`~object.__enter__` and :meth:`~object.__exit__` methods). -The object's :meth:`__enter__` is called before *with-block* is executed and +The object's :meth:`~object.__enter__` is called before *with-block* is executed and therefore can run set-up code. It also may return a value that is bound to the name *variable*, if given. (Note carefully that *variable* is *not* assigned the result of *expression*.) -After execution of the *with-block* is finished, the object's :meth:`__exit__` +After execution of the *with-block* is finished, the object's :meth:`~object.__exit__` method is called, even if the block raised an exception, and can therefore run clean-up code. @@ -296,7 +296,7 @@ part-way through the block. .. note:: In this case, *f* is the same object created by :func:`open`, because - :meth:`file.__enter__` returns *self*. + :meth:`~object.__enter__` returns *self*. The :mod:`threading` module's locks and condition variables also support the ':keyword:`with`' statement:: @@ -339,16 +339,16 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`__enter__` and :meth:`__exit__` + manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. -* The context manager's :meth:`__enter__` method is called. The value returned +* The context manager's :meth:`~object.__enter__` method is called. The value returned is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply discarded. * The code in *BLOCK* is executed. -* If *BLOCK* raises an exception, the context manager's :meth:`__exit__` method +* If *BLOCK* raises an exception, the context manager's :meth:`~object.__exit__` method is called with three arguments, the exception details (``type, value, traceback``, the same values returned by :func:`sys.exc_info`, which can also be ``None`` if no exception occurred). The method's return value controls whether an exception @@ -357,7 +357,7 @@ A high-level explanation of the context management protocol is: if you do the author of the code containing the ':keyword:`with`' statement will never realize anything went wrong. -* If *BLOCK* didn't raise an exception, the :meth:`__exit__` method is still +* If *BLOCK* didn't raise an exception, the :meth:`~object.__exit__` method is still called, but *type*, *value*, and *traceback* are all ``None``. Let's think through an example. I won't present detailed code but will only @@ -391,7 +391,7 @@ rolled back if there's an exception. Here's the basic interface for def rollback(self): "Rolls back current transaction" -The :meth:`__enter__` method is pretty easy, having only to start a new +The :meth:`~object.__enter__` method is pretty easy, having only to start a new transaction. For this application the resulting cursor object would be a useful result, so the method will return it. The user can then add ``as cursor`` to their ':keyword:`with`' statement to bind the cursor to a variable name. :: @@ -403,7 +403,7 @@ their ':keyword:`with`' statement to bind the cursor to a variable name. :: cursor = self.cursor() return cursor -The :meth:`__exit__` method is the most complicated because it's where most of +The :meth:`~object.__exit__` method is the most complicated because it's where most of the work has to be done. The method has to check if an exception occurred. If there was no exception, the transaction is committed. The transaction is rolled back if there was an exception. @@ -436,10 +436,10 @@ are useful when writing objects for use with the ':keyword:`with`' statement. The decorator is called :func:`contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`__enter__` method, and the value yielded will be the method's return +:meth:`~object.__enter__` method, and the value yielded will be the method's return value that will get bound to the variable in the ':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`__exit__` method. Any exception raised in the block will +executed in the :meth:`~object.__exit__` method. Any exception raised in the block will be raised by the :keyword:`!yield` statement. Using this decorator, our database example from the previous section @@ -1737,7 +1737,7 @@ Optimizations (Contributed by Antoine Pitrou.) Memory usage is reduced by using pymalloc for the Unicode string's data. -* The ``with`` statement now stores the :meth:`__exit__` method on the stack, +* The ``with`` statement now stores the :meth:`~object.__exit__` method on the stack, producing a small speedup. (Implemented by Jeffrey Yasskin.) * To reduce memory usage, the garbage collector will now clear internal diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index f45d45f5b5bf721..61934ab1a2df567 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -930,8 +930,8 @@ Optimizations Several performance enhancements have been added: * A new opcode was added to perform the initial setup for - :keyword:`with` statements, looking up the :meth:`__enter__` and - :meth:`__exit__` methods. (Contributed by Benjamin Peterson.) + :keyword:`with` statements, looking up the :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. (Contributed by Benjamin Peterson.) * The garbage collector now performs better for one common usage pattern: when many objects are being allocated without deallocating @@ -2449,13 +2449,13 @@ that may require changes to your code: (Changed by Eric Smith; :issue:`5920`.) * Because of an optimization for the :keyword:`with` statement, the special - methods :meth:`__enter__` and :meth:`__exit__` must belong to the object's + methods :meth:`~object.__enter__` and :meth:`~object.__exit__` must belong to the object's type, and cannot be directly attached to the object's instance. This affects new-style classes (derived from :class:`object`) and C extension types. (:issue:`6101`.) * Due to a bug in Python 2.6, the *exc_value* parameter to - :meth:`__exit__` methods was often the string representation of the + :meth:`~object.__exit__` methods was often the string representation of the exception, not an instance. This was fixed in 2.7, so *exc_value* will be an instance as expected. (Fixed by Florent Xicluna; :issue:`7853`.) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 77fcb8b77e45c74..3bf4f96eafa15bf 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -117,6 +117,10 @@ Other Language Changes the file is not accessible. (Contributed by Moonsik Park in :gh:`82367`.) +* Fixed a bug where a :keyword:`global` decleration in an :keyword:`except` block + is rejected when the global is used in the :keyword:`else` block. + (Contributed by Irit Katriel in :gh:`111123`.) + New Modules =========== @@ -959,8 +963,9 @@ Build Changes library, GCC built-in atomic functions, or MSVC interlocked intrinsics. * The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, - ``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions - are now built with the :ref:`limited C API `. + ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_testimportmultiple`` and ``_uuid`` C extensions are now built with the + :ref:`limited C API `. (Contributed by Victor Stinner in :gh:`85283`.) @@ -1089,6 +1094,9 @@ New Features limited C API. (Contributed by Victor Stinner in :gh:`85283`.) +* Add :c:func:`PyUnicode_AsUTF8` function to the limited C API. + (Contributed by Victor Stinner in :gh:`111089`.) + Porting to Python 3.13 ---------------------- @@ -1158,6 +1166,12 @@ Porting to Python 3.13 Note that ``Py_TRASHCAN_BEGIN`` has a second argument which should be the deallocation function it is in. +* The :c:func:`PyUnicode_AsUTF8` function now raises an exception if the string + contains embedded null characters. To accept embedded null characters and + truncate on purpose at the first null byte, + ``PyUnicode_AsUTF8AndSize(unicode, NULL)`` can be used instead. + (Contributed by Victor Stinner in :gh:`111089`.) + Deprecated ---------- diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index a2a67264deff662..4574702b1a600f7 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1653,7 +1653,7 @@ Deprecated deprecated and will be prohibited in Python 3.9. (Contributed by Elvis Pranskevichus in :issue:`34075`.) -* The :meth:`__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, +* The :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput` have been deprecated. diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 859ab7178e920ac..d200fa0622cef59 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -440,19 +440,6 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromKindAndData( const void *buffer, Py_ssize_t size); -/* --- Manage the default encoding ---------------------------------------- */ - -/* Returns a pointer to the default encoding (UTF-8) of the - Unicode object unicode. - - Like PyUnicode_AsUTF8AndSize(), this also caches the UTF-8 representation - in the unicodeobject. - - Use of this API is DEPRECATED since no size information can be - extracted from the returned data. -*/ - -PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); /* === Characters Type APIs =============================================== */ diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index dee00715b3c51d5..ee7b769ce5a6fc2 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -443,17 +443,25 @@ PyAPI_FUNC(PyObject*) PyUnicode_AsUTF8String( PyObject *unicode /* Unicode object */ ); -/* Returns a pointer to the default encoding (UTF-8) of the - Unicode object unicode and the size of the encoded representation - in bytes stored in *size. - - In case of an error, no *size is set. - - This function caches the UTF-8 encoded string in the unicodeobject - and subsequent calls will return the same string. The memory is released - when the unicodeobject is deallocated. -*/ - +// Returns a pointer to the UTF-8 encoding of the Unicode object unicode. +// +// Raise an exception if the string contains embedded null characters. +// Use PyUnicode_AsUTF8AndSize() to accept embedded null characters. +// +// This function caches the UTF-8 encoded string in the Unicode object +// and subsequent calls will return the same string. The memory is released +// when the Unicode object is deallocated. +PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); + +// Returns a pointer to the UTF-8 encoding of the +// Unicode object unicode and the size of the encoded representation +// in bytes stored in `*size` (if size is not NULL). +// +// On error, `*size` is set to 0 (if size is not NULL). +// +// This function caches the UTF-8 encoded string in the Unicode object +// and subsequent calls will return the same string. The memory is released +// when the Unicode object is deallocated. #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize( PyObject *unicode, diff --git a/Lib/code.py b/Lib/code.py index 2bd5fa3e795a619..f4aecddeca7813b 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -5,6 +5,7 @@ # Inspired by similar code by Jeff Epler and Fredrik Lundh. +import builtins import sys import traceback from codeop import CommandCompiler, compile_command @@ -169,7 +170,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename=""): + def __init__(self, locals=None, filename="", local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -181,6 +182,7 @@ def __init__(self, locals=None, filename=""): """ InteractiveInterpreter.__init__(self, locals) self.filename = filename + self.local_exit = local_exit self.resetbuffer() def resetbuffer(self): @@ -219,27 +221,64 @@ def interact(self, banner=None, exitmsg=None): elif banner: self.write("%s\n" % str(banner)) more = 0 - while 1: - try: - if more: - prompt = sys.ps2 - else: - prompt = sys.ps1 + + # When the user uses exit() or quit() in their interactive shell + # they probably just want to exit the created shell, not the whole + # process. exit and quit in builtins closes sys.stdin which makes + # it super difficult to restore + # + # When self.local_exit is True, we overwrite the builtins so + # exit() and quit() only raises SystemExit and we can catch that + # to only exit the interactive shell + + _exit = None + _quit = None + + if self.local_exit: + if hasattr(builtins, "exit"): + _exit = builtins.exit + builtins.exit = Quitter("exit") + + if hasattr(builtins, "quit"): + _quit = builtins.quit + builtins.quit = Quitter("quit") + + try: + while True: try: - line = self.raw_input(prompt) - except EOFError: - self.write("\n") - break - else: - more = self.push(line) - except KeyboardInterrupt: - self.write("\nKeyboardInterrupt\n") - self.resetbuffer() - more = 0 - if exitmsg is None: - self.write('now exiting %s...\n' % self.__class__.__name__) - elif exitmsg != '': - self.write('%s\n' % exitmsg) + if more: + prompt = sys.ps2 + else: + prompt = sys.ps1 + try: + line = self.raw_input(prompt) + except EOFError: + self.write("\n") + break + else: + more = self.push(line) + except KeyboardInterrupt: + self.write("\nKeyboardInterrupt\n") + self.resetbuffer() + more = 0 + except SystemExit as e: + if self.local_exit: + self.write("\n") + break + else: + raise e + finally: + # restore exit and quit in builtins if they were modified + if _exit is not None: + builtins.exit = _exit + + if _quit is not None: + builtins.quit = _quit + + if exitmsg is None: + self.write('now exiting %s...\n' % self.__class__.__name__) + elif exitmsg != '': + self.write('%s\n' % exitmsg) def push(self, line): """Push a line to the interpreter. @@ -276,8 +315,22 @@ def raw_input(self, prompt=""): return input(prompt) +class Quitter: + def __init__(self, name): + self.name = name + if sys.platform == "win32": + self.eof = 'Ctrl-Z plus Return' + else: + self.eof = 'Ctrl-D (i.e. EOF)' + + def __repr__(self): + return f'Use {self.name} or {self.eof} to exit' + + def __call__(self, code=None): + raise SystemExit(code) + -def interact(banner=None, readfunc=None, local=None, exitmsg=None): +def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False): """Closely emulate the interactive Python interpreter. This is a backwards compatible interface to the InteractiveConsole @@ -290,9 +343,10 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None): readfunc -- if not None, replaces InteractiveConsole.raw_input() local -- passed to InteractiveInterpreter.__init__() exitmsg -- passed to InteractiveConsole.interact() + local_exit -- passed to InteractiveConsole.__init__() """ - console = InteractiveConsole(local) + console = InteractiveConsole(local, local_exit=local_exit) if readfunc is not None: console.raw_input = readfunc else: diff --git a/Lib/doctest.py b/Lib/doctest.py index 46b4dd6769b063f..f00d9358ffe10be 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1393,7 +1393,20 @@ def __run(self, test, compileflags, out): # The example raised an exception: check if it was expected. else: - exc_msg = traceback.format_exception_only(*exception[:2])[-1] + formatted_ex = traceback.format_exception_only(*exception[:2]) + if issubclass(exception[0], SyntaxError): + # SyntaxError / IndentationError is special: + # we don't care about the carets / suggestions / etc + # We only care about the error message and notes. + # They start with `SyntaxError:` (or any other class name) + exc_msg_index = next( + index + for index, line in enumerate(formatted_ex) + if line.startswith(f"{exception[0].__name__}:") + ) + formatted_ex = formatted_ex[exc_msg_index:] + + exc_msg = "".join(formatted_ex) if not quiet: got += _exception_traceback(exception) diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 2b09d79470b47c4..898efeb4dd15508 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -597,7 +597,9 @@ def GetCoreKeys(self, keySetName=None): problem getting any core binding there will be an 'ultimate last resort fallback' to the CUA-ish bindings defined here. """ + # TODO: = dict(sorted([(v-event, keys), ...]))? keyBindings={ + # vitual-event: list of key events. '<>': ['', ''], '<>': ['', ''], '<>': ['', ''], @@ -880,7 +882,7 @@ def _dump(): # htest # (not really, but ignore in coverage) line, crc = 0, 0 def sprint(obj): - global line, crc + nonlocal line, crc txt = str(obj) line += 1 crc = crc32(txt.encode(encoding='utf-8'), crc) @@ -889,7 +891,7 @@ def sprint(obj): def dumpCfg(cfg): print('\n', cfg, '\n') # Cfg has variable '0xnnnnnnnn' address. - for key in sorted(cfg.keys()): + for key in sorted(cfg): sections = cfg[key].sections() sprint(key) sprint(sections) @@ -908,4 +910,6 @@ def dumpCfg(cfg): from unittest import main main('idlelib.idle_test.test_config', verbosity=2, exit=False) - # Run revised _dump() as htest? + _dump() + # Run revised _dump() (700+ lines) as htest? More sorting. + # Perhaps as window with tabs for textviews, making it config viewer. diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index cda7966d558a51c..eedf97bf74fe6af 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -211,14 +211,8 @@ def help(self): contents=help_common+help_pages.get(page, '')) def deactivate_current_config(self): - """Remove current key bindings. - Iterate over window instances defined in parent and remove - the keybindings. - """ - # Before a config is saved, some cleanup of current - # config must be done - remove the previous keybindings. - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + """Remove current key bindings in current windows.""" + for instance in self.parent.instance_dict: instance.RemoveKeybindings() def activate_config_changes(self): @@ -227,8 +221,7 @@ def activate_config_changes(self): Dynamically update the current parent window instances with some of the configuration changes. """ - win_instances = self.parent.instance_dict.keys() - for instance in win_instances: + for instance in self.parent.instance_dict: instance.ResetColorizer() instance.ResetFont() instance.set_notabs_indentwidth() @@ -583,22 +576,23 @@ def create_page_highlight(self): (*)theme_message: Label """ self.theme_elements = { - 'Normal Code or Text': ('normal', '00'), - 'Code Context': ('context', '01'), - 'Python Keywords': ('keyword', '02'), - 'Python Definitions': ('definition', '03'), - 'Python Builtins': ('builtin', '04'), - 'Python Comments': ('comment', '05'), - 'Python Strings': ('string', '06'), - 'Selected Text': ('hilite', '07'), - 'Found Text': ('hit', '08'), - 'Cursor': ('cursor', '09'), - 'Editor Breakpoint': ('break', '10'), - 'Shell Prompt': ('console', '11'), - 'Error Text': ('error', '12'), - 'Shell User Output': ('stdout', '13'), - 'Shell User Exception': ('stderr', '14'), - 'Line Number': ('linenumber', '16'), + # Display-name: internal-config-tag-name. + 'Normal Code or Text': 'normal', + 'Code Context': 'context', + 'Python Keywords': 'keyword', + 'Python Definitions': 'definition', + 'Python Builtins': 'builtin', + 'Python Comments': 'comment', + 'Python Strings': 'string', + 'Selected Text': 'hilite', + 'Found Text': 'hit', + 'Cursor': 'cursor', + 'Editor Breakpoint': 'break', + 'Shell Prompt': 'console', + 'Error Text': 'error', + 'Shell User Output': 'stdout', + 'Shell User Exception': 'stderr', + 'Line Number': 'linenumber', } self.builtin_name = tracers.add( StringVar(self), self.var_changed_builtin_name) @@ -658,7 +652,7 @@ def tem(event, elem=element): # event.widget.winfo_top_level().highlight_target.set(elem) self.highlight_target.set(elem) text.tag_bind( - self.theme_elements[element][0], '', tem) + self.theme_elements[element], '', tem) text['state'] = 'disabled' self.style.configure('frame_color_set.TFrame', borderwidth=1, relief='solid') @@ -765,8 +759,7 @@ def load_theme_cfg(self): self.builtinlist.SetMenu(item_list, item_list[0]) self.set_theme_type() # Load theme element option menu. - theme_names = list(self.theme_elements.keys()) - theme_names.sort(key=lambda x: self.theme_elements[x][1]) + theme_names = list(self.theme_elements) self.targetlist.SetMenu(theme_names, theme_names[0]) self.paint_theme_sample() self.set_highlight_target() @@ -893,7 +886,7 @@ def on_new_color_set(self): new_color = self.color.get() self.style.configure('frame_color_set.TFrame', background=new_color) plane = 'foreground' if self.fg_bg_toggle.get() else 'background' - sample_element = self.theme_elements[self.highlight_target.get()][0] + sample_element = self.theme_elements[self.highlight_target.get()] self.highlight_sample.tag_config(sample_element, **{plane: new_color}) theme = self.custom_name.get() theme_element = sample_element + '-' + plane @@ -1007,7 +1000,7 @@ def set_color_sample(self): frame_color_set """ # Set the color sample area. - tag = self.theme_elements[self.highlight_target.get()][0] + tag = self.theme_elements[self.highlight_target.get()] plane = 'foreground' if self.fg_bg_toggle.get() else 'background' color = self.highlight_sample.tag_cget(tag, plane) self.style.configure('frame_color_set.TFrame', background=color) @@ -1037,7 +1030,7 @@ def paint_theme_sample(self): else: # User theme theme = self.custom_name.get() for element_title in self.theme_elements: - element = self.theme_elements[element_title][0] + element = self.theme_elements[element_title] colors = idleConf.GetHighlight(theme, element) if element == 'cursor': # Cursor sample needs special painting. colors['background'] = idleConf.GetHighlight( @@ -1477,12 +1470,13 @@ def load_keys_list(self, keyset_name): reselect = True list_index = self.bindingslist.index(ANCHOR) keyset = idleConf.GetKeySet(keyset_name) - bind_names = list(keyset.keys()) + # 'set' is dict mapping virtual event to list of key events. + bind_names = list(keyset) bind_names.sort() self.bindingslist.delete(0, END) for bind_name in bind_names: key = ' '.join(keyset[bind_name]) - bind_name = bind_name[2:-2] # Trim off the angle brackets. + bind_name = bind_name[2:-2] # Trim double angle brackets. if keyset_name in changes['keys']: # Handle any unsaved changes to this key set. if bind_name in changes['keys'][keyset_name]: diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index 452c62b42655b3b..a92bb98d908d462 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -509,7 +509,7 @@ def load_dict(self, dict, force=0, rpc_client=None): # There is also an obscure bug in sorted(dict) where the # interpreter gets into a loop requesting non-existing dict[0], # dict[1], dict[2], etc from the debugger_r.DictProxy. - ### + # TODO recheck above; see debugger_r 159ff, debugobj 60. keys_list = dict.keys() names = sorted(keys_list) ### diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 71d01c7070df54c..032b686f379378b 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -93,7 +93,8 @@ def setfunction(value, key=key, object=self.object): class DictTreeItem(SequenceTreeItem): def keys(self): - keys = list(self.object.keys()) + # TODO return sorted(self.object) + keys = list(self.object) try: keys.sort() except: diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 08ed76fe288294c..a746f1538a62b00 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -274,8 +274,8 @@ def test_create_config_handlers(self): conf.CreateConfigHandlers() # Check keys are equal - self.assertCountEqual(conf.defaultCfg.keys(), conf.config_types) - self.assertCountEqual(conf.userCfg.keys(), conf.config_types) + self.assertCountEqual(conf.defaultCfg, conf.config_types) + self.assertCountEqual(conf.userCfg, conf.config_types) # Check conf parser are correct type for default_parser in conf.defaultCfg.values(): diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index e5d5b4013fca571..6f8518a9bb19d01 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -430,7 +430,7 @@ def test_highlight_target_text_mouse(self): def tag_to_element(elem): for element, tag in d.theme_elements.items(): - elem[tag[0]] = element + elem[tag] = element def click_it(start): x, y, dx, dy = hs.bbox(start) diff --git a/Lib/idlelib/idle_test/test_debugobj.py b/Lib/idlelib/idle_test/test_debugobj.py index 131ce22b8bb69b0..90ace4e1bc4f9ef 100644 --- a/Lib/idlelib/idle_test/test_debugobj.py +++ b/Lib/idlelib/idle_test/test_debugobj.py @@ -37,7 +37,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.SequenceTreeItem('label', 'abc') - self.assertEqual(list(ti.keys()), [0, 1, 2]) + self.assertEqual(list(ti.keys()), [0, 1, 2]) # keys() is a range. class DictTreeItemTest(unittest.TestCase): @@ -50,7 +50,7 @@ def test_isexpandable(self): def test_keys(self): ti = debugobj.DictTreeItem('label', {1:1, 0:0, 2:2}) - self.assertEqual(ti.keys(), [0, 1, 2]) + self.assertEqual(ti.keys(), [0, 1, 2]) # keys() is a sorted list. if __name__ == '__main__': diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 6028700356b1718..7a2707935b60c93 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -747,10 +747,11 @@ def showtraceback(self): self.tkconsole.open_stack_viewer() def checklinecache(self): - c = linecache.cache - for key in list(c.keys()): + "Remove keys other than ''." + cache = linecache.cache + for key in list(cache): # Iterate list because mutate cache. if key[:1] + key[-1:] != "<>": - del c[key] + del cache[key] def runcommand(self, code): "Run the code without invoking the debugger" diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 7b00c4cdb7d0338..4858cc682a4f453 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -99,7 +99,7 @@ def IsExpandable(self): def GetSubList(self): sublist = [] - for key in self.object.keys(): + for key in self.object.keys(): # self.object not necessarily dict. try: value = self.object[key] except KeyError: diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index f4d6e82331516f9..3ad71d31c2f4384 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -135,7 +135,7 @@ class _incompatible_extension_module_restrictions: may not be imported in a subinterpreter. That implies modules that do not implement multi-phase init or that explicitly of out. - Likewise for modules import in a subinterpeter with its own GIL + Likewise for modules import in a subinterpreter with its own GIL when the extension does not support a per-interpreter GIL. This implies the module does not have a Py_mod_multiple_interpreters slot set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. diff --git a/Lib/pdb.py b/Lib/pdb.py index 67f8d57c1a7768f..1e4d0a20515fa33 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1741,7 +1741,7 @@ def do_interact(self, arg): contains all the (global and local) names found in the current scope. """ ns = {**self.curframe.f_globals, **self.curframe_locals} - code.interact("*interactive*", local=ns) + code.interact("*interactive*", local=ns, local_exit=True) def do_alias(self, arg): """alias [name [command]] diff --git a/Lib/socket.py b/Lib/socket.py index 321fcda51505c18..5f0a1f40e25b941 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -702,16 +702,15 @@ def readinto(self, b): self._checkReadable() if self._timeout_occurred: raise OSError("cannot read from timed out object") - while True: - try: - return self._sock.recv_into(b) - except timeout: - self._timeout_occurred = True - raise - except error as e: - if e.errno in _blocking_errnos: - return None - raise + try: + return self._sock.recv_into(b) + except timeout: + self._timeout_occurred = True + raise + except error as e: + if e.errno in _blocking_errnos: + return None + raise def write(self, b): """Write the given bytes or bytearray object *b* to the socket diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 152b558808e66a6..87b926db0686cec 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -161,8 +161,7 @@ def __init__(self, **kwargs) -> None: self.forever = False self.header = False self.failfast = False - self.match_tests = None - self.ignore_tests = None + self.match_tests = [] self.pgo = False self.pgo_extended = False self.worker_json = None @@ -183,6 +182,20 @@ def error(self, message): super().error(message + "\nPass -h or --help for complete help.") +class FilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + items.append((value, self.const)) + + +class FromFileFilterAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + items = getattr(namespace, self.dest) + with open(value, encoding='utf-8') as fp: + for line in fp: + items.append((line.strip(), self.const)) + + def _create_parser(): # Set prog to prevent the uninformative "__main__.py" from displaying in # error messages when using "python -m test ...". @@ -192,6 +205,7 @@ def _create_parser(): epilog=EPILOG, add_help=False, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.set_defaults(match_tests=[]) # Arguments with this clause added to its help are described further in # the epilog's "Additional option details" section. @@ -251,17 +265,19 @@ def _create_parser(): help='single step through a set of tests.' + more_details) group.add_argument('-m', '--match', metavar='PAT', - dest='match_tests', action='append', + dest='match_tests', action=FilterAction, const=True, help='match test cases and methods with glob pattern PAT') group.add_argument('-i', '--ignore', metavar='PAT', - dest='ignore_tests', action='append', + dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern PAT') group.add_argument('--matchfile', metavar='FILENAME', - dest='match_filename', + dest='match_tests', + action=FromFileFilterAction, const=True, help='similar to --match but get patterns from a ' 'text file, one pattern per line') group.add_argument('--ignorefile', metavar='FILENAME', - dest='ignore_filename', + dest='match_tests', + action=FromFileFilterAction, const=False, help='similar to --matchfile but it receives patterns ' 'from text file to ignore') group.add_argument('-G', '--failfast', action='store_true', @@ -482,18 +498,6 @@ def _parse_args(args, **kwargs): print("WARNING: Disable --verbose3 because it's incompatible with " "--huntrleaks: see http://bugs.python.org/issue27103", file=sys.stderr) - if ns.match_filename: - if ns.match_tests is None: - ns.match_tests = [] - with open(ns.match_filename) as fp: - for line in fp: - ns.match_tests.append(line.strip()) - if ns.ignore_filename: - if ns.ignore_tests is None: - ns.ignore_tests = [] - with open(ns.ignore_filename) as fp: - for line in fp: - ns.ignore_tests.append(line.strip()) if ns.forever: # --forever implies --failfast ns.failfast = True diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index caf2872cfc9a18f..f3ff3628bea1341 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -5,7 +5,7 @@ from test import support from .utils import ( - StrPath, TestName, TestTuple, TestList, FilterTuple, + StrPath, TestName, TestTuple, TestList, TestFilter, abs_module_name, count, printlist) @@ -83,11 +83,10 @@ def _list_cases(suite): print(test.id()) def list_cases(tests: TestTuple, *, - match_tests: FilterTuple | None = None, - ignore_tests: FilterTuple | None = None, + match_tests: TestFilter | None = None, test_dir: StrPath | None = None): support.verbose = False - support.set_match_tests(match_tests, ignore_tests) + support.set_match_tests(match_tests) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index cb60d5af732b437..9b86548c89fb2e2 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -19,7 +19,7 @@ from .setup import setup_process, setup_test_dir from .single import run_single_test, PROGRESS_MIN_TIME from .utils import ( - StrPath, StrJSON, TestName, TestList, TestTuple, FilterTuple, + StrPath, StrJSON, TestName, TestList, TestTuple, TestFilter, strip_py_suffix, count, format_duration, printlist, get_temp_dir, get_work_dir, exit_timeout, display_header, cleanup_temp_dir, print_warning, @@ -78,14 +78,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): and ns._add_python_opts) # Select tests - if ns.match_tests: - self.match_tests: FilterTuple | None = tuple(ns.match_tests) - else: - self.match_tests = None - if ns.ignore_tests: - self.ignore_tests: FilterTuple | None = tuple(ns.ignore_tests) - else: - self.ignore_tests = None + self.match_tests: TestFilter = ns.match_tests self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start @@ -129,14 +122,19 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Randomize self.randomize: bool = ns.randomize - self.random_seed: int | None = ( - ns.random_seed - if ns.random_seed is not None - else random.getrandbits(32) - ) - if 'SOURCE_DATE_EPOCH' in os.environ: + if ('SOURCE_DATE_EPOCH' in os.environ + # don't use the variable if empty + and os.environ['SOURCE_DATE_EPOCH'] + ): self.randomize = False - self.random_seed = None + # SOURCE_DATE_EPOCH should be an integer, but use a string to not + # fail if it's not integer. random.seed() accepts a string. + # https://reproducible-builds.org/docs/source-date-epoch/ + self.random_seed: int | str = os.environ['SOURCE_DATE_EPOCH'] + elif ns.random_seed is None: + self.random_seed = random.getrandbits(32) + else: + self.random_seed = ns.random_seed # tests self.first_runtests: RunTests | None = None @@ -384,7 +382,7 @@ def finalize_tests(self, tracer): def display_summary(self): duration = time.perf_counter() - self.logger.start_time - filtered = bool(self.match_tests) or bool(self.ignore_tests) + filtered = bool(self.match_tests) # Total duration print() @@ -402,7 +400,6 @@ def create_run_tests(self, tests: TestTuple): fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, match_tests_dict=None, rerun=False, forever=self.forever, @@ -441,7 +438,7 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: or tests or self.cmdline_args)): display_header(self.use_resources, self.python_cmd) - print("Using random seed", self.random_seed) + print("Using random seed:", self.random_seed) runtests = self.create_run_tests(selected) self.first_runtests = runtests @@ -655,7 +652,6 @@ def main(self, tests: TestList | None = None): elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, - ignore_tests=self.ignore_tests, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 16f8331abd32f9d..ab03cb54d6122ef 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -261,7 +261,7 @@ def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> Ru kwargs = {} if match_tests: - kwargs['match_tests'] = match_tests + kwargs['match_tests'] = [(test, True) for test in match_tests] if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 4da312db4cb02e5..bfed1b4a2a58170 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -8,7 +8,7 @@ from test import support from .utils import ( - StrPath, StrJSON, TestTuple, FilterTuple, FilterDict) + StrPath, StrJSON, TestTuple, TestFilter, FilterTuple, FilterDict) class JsonFileType: @@ -72,8 +72,7 @@ class RunTests: tests: TestTuple fail_fast: bool fail_env_changed: bool - match_tests: FilterTuple | None - ignore_tests: FilterTuple | None + match_tests: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool @@ -91,7 +90,7 @@ class RunTests: use_resources: tuple[str, ...] python_cmd: tuple[str, ...] | None randomize: bool - random_seed: int | None + random_seed: int | str json_file: JsonFile | None def copy(self, **override): diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 793347f60ad93c1..6a96b051394d208 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -92,7 +92,7 @@ def setup_tests(runtests: RunTests): support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - support.set_match_tests(runtests.match_tests, runtests.ignore_tests) + support.set_match_tests(runtests.match_tests) if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index aac8395da87845b..bd4dce3400f0a93 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -52,6 +52,7 @@ TestList = list[TestName] # --match and --ignore options: list of patterns # ('*' joker character can be used) +TestFilter = list[tuple[TestName, bool]] FilterTuple = tuple[TestName, ...] FilterDict = dict[TestName, FilterTuple] diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index a9c8be0bb65d087..2eccfabc25223af 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -10,7 +10,7 @@ from .runtests import RunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( - StrPath, StrJSON, FilterTuple, + StrPath, StrJSON, TestFilter, get_temp_dir, get_work_dir, exit_timeout) @@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: runtests = RunTests.from_json(worker_json) test_name = runtests.tests[0] - match_tests: FilterTuple | None = runtests.match_tests + match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file setup_test_dir(runtests.test_dir) @@ -81,7 +81,7 @@ def worker_process(worker_json: StrJSON) -> NoReturn: if runtests.rerun: if match_tests: - matching = "matching: " + ", ".join(match_tests) + matching = "matching: " + ", ".join(pattern for pattern, result in match_tests if result) print(f"Re-running {test_name} in verbose mode ({matching})", flush=True) else: print(f"Re-running {test_name} in verbose mode", flush=True) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 21e8770ab31180e..695ffd04e91fb2f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,8 +6,10 @@ import contextlib import dataclasses import functools +import itertools import getpass import _opcode +import operator import os import re import stat @@ -1183,18 +1185,17 @@ def _run_suite(suite): # By default, don't filter tests -_match_test_func = None - -_accept_test_patterns = None -_ignore_test_patterns = None +_test_matchers = () +_test_patterns = () def match_test(test): # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) + result = False + for matcher, result in reversed(_test_matchers): + if matcher(test.id()): + return result + return not result def _is_full_match_test(pattern): @@ -1207,47 +1208,30 @@ def _is_full_match_test(pattern): return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns - - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () - - accept_func = ignore_func = None - - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) +def set_match_tests(patterns): + global _test_matchers, _test_patterns - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function + if not patterns: + _test_matchers = () + _test_patterns = () + else: + itemgetter = operator.itemgetter + patterns = tuple(patterns) + if patterns != _test_patterns: + _test_matchers = [ + (_compile_match_function(map(itemgetter(0), it)), result) + for result, it in itertools.groupby(patterns, itemgetter(1)) + ] + _test_patterns = patterns def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): + patterns = list(patterns) + + if all(map(_is_full_match_test, patterns)): # Simple case: all patterns are full test identifier. # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ + return set(patterns).__contains__ else: import fnmatch regex = '|'.join(map(fnmatch.translate, patterns)) @@ -1255,7 +1239,7 @@ def _compile_match_function(patterns): # don't use flags=re.IGNORECASE regex_match = re.compile(regex).match - def match_test_regex(test_id): + def match_test_regex(test_id, regex_match=regex_match): if regex_match(test_id): # The regex matches the whole identifier, for example # 'test.test_os.FileTests.test_access'. @@ -1266,9 +1250,7 @@ def match_test_regex(test_id): # into: 'test', 'test_os', 'FileTests' and 'test_access'. return any(map(regex_match, test_id.split("."))) - func = match_test_regex - - return patterns, func + return match_test_regex def run_unittest(*classes): diff --git a/Lib/test/support/interpreters.py b/Lib/test/support/interpreters.py index f8f42c0e02479cb..182f47b19f1dd4f 100644 --- a/Lib/test/support/interpreters.py +++ b/Lib/test/support/interpreters.py @@ -161,6 +161,14 @@ def __eq__(self, other): def id(self): return self._id + @property + def _info(self): + return _channels.get_info(self._id) + + @property + def is_closed(self): + return self._info.closed + _NOT_SET = object() @@ -213,6 +221,11 @@ class SendChannel(_ChannelEnd): _end = 'send' + @property + def is_closed(self): + info = self._info + return info.closed or info.closing + def send(self, obj, timeout=None): """Send the object (i.e. its data) to the channel's receiving end. @@ -251,4 +264,4 @@ def close(self): # XXX This is causing leaks (gh-110318): -#_channels._register_end_types(SendChannel, RecvChannel) +_channels._register_end_types(SendChannel, RecvChannel) diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index ef744f6b97259ce..82dea8a6d731ea3 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -474,6 +474,12 @@ def test_base64_roundtrip(self, binary, newline): restored = binascii.a2b_base64(self.type2test(converted)) self.assertConversion(binary, converted, restored, newline=newline) + def test_c_contiguity(self): + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + binascii.b2a_hex(noncontig_writable) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 96d34ab94c5981e..6ff436993e59811 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -153,6 +153,8 @@ class TupleSubclass(tuple): class DictSubclass(dict): pass +NONCONTIG_WRITABLE = memoryview(bytearray(b'noncontig'))[::-2] +NONCONTIG_READONLY = memoryview(b'noncontig')[::-2] class Unsigned_TestCase(unittest.TestCase): def test_b(self): @@ -837,6 +839,8 @@ def test_y_star(self): self.assertEqual(getargs_y_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_y_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_y_star, None) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_y_star, NONCONTIG_READONLY) def test_y_hash(self): from _testcapi import getargs_y_hash @@ -846,6 +850,9 @@ def test_y_hash(self): self.assertRaises(TypeError, getargs_y_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_y_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_y_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_y_hash, NONCONTIG_READONLY) def test_w_star(self): # getargs_w_star() modifies first and last byte @@ -861,6 +868,8 @@ def test_w_star(self): self.assertEqual(getargs_w_star(memoryview(buf)), b'[emoryvie]') self.assertEqual(buf, bytearray(b'[emoryvie]')) self.assertRaises(TypeError, getargs_w_star, None) + self.assertRaises(TypeError, getargs_w_star, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_w_star, NONCONTIG_READONLY) class String_TestCase(unittest.TestCase): @@ -893,6 +902,8 @@ def test_s_star(self): self.assertEqual(getargs_s_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_s_star(memoryview(b'memoryview')), b'memoryview') self.assertRaises(TypeError, getargs_s_star, None) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_s_star, NONCONTIG_READONLY) def test_s_hash(self): from _testcapi import getargs_s_hash @@ -902,6 +913,9 @@ def test_s_hash(self): self.assertRaises(TypeError, getargs_s_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_s_hash, memoryview(b'memoryview')) self.assertRaises(TypeError, getargs_s_hash, None) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_s_hash, NONCONTIG_READONLY) def test_z(self): from _testcapi import getargs_z @@ -920,6 +934,8 @@ def test_z_star(self): self.assertEqual(getargs_z_star(bytearray(b'bytearray')), b'bytearray') self.assertEqual(getargs_z_star(memoryview(b'memoryview')), b'memoryview') self.assertIsNone(getargs_z_star(None)) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_WRITABLE) + self.assertRaises(BufferError, getargs_z_star, NONCONTIG_READONLY) def test_z_hash(self): from _testcapi import getargs_z_hash @@ -929,6 +945,9 @@ def test_z_hash(self): self.assertRaises(TypeError, getargs_z_hash, bytearray(b'bytearray')) self.assertRaises(TypeError, getargs_z_hash, memoryview(b'memoryview')) self.assertIsNone(getargs_z_hash(None)) + # TypeError: must be read-only bytes-like object, not memoryview + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_WRITABLE) + self.assertRaises(TypeError, getargs_z_hash, NONCONTIG_READONLY) def test_es(self): from _testcapi import getargs_es diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index a73e669dda7ddce..8ab55902b1fd12f 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -882,7 +882,10 @@ def test_asutf8(self): self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') - self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') + + # disallow embedded null characters + self.assertRaises(ValueError, unicode_asutf8, 'abc\0', 0) + self.assertRaises(ValueError, unicode_asutf8, 'abc\0def', 0) self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) @@ -906,7 +909,11 @@ def test_asutf8andsize(self): self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize_null, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize_null, [], 0) # CRASHES unicode_asutf8andsize(NULL, 0) + # CRASHES unicode_asutf8andsize_null(NULL, 0) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 614c6b3c3b52994..e9405ff8b214145 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -684,6 +684,17 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) + def test_syntaxerror_does_not_crash(self): + script = "nonlocal x\n" + with os_helper.temp_dir() as script_dir: + script_name = _make_test_script(script_dir, 'script', script) + exitcode, stdout, stderr = assert_python_failure(script_name) + text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() + # It used to crash in https://github.com/python/cpython/issues/111132 + self.assertTrue(text.endswith( + 'SyntaxError: nonlocal declaration not allowed at module level\n', + ), text) + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 226bc3a853b7b94..747c0f9683c19c7 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -10,11 +10,7 @@ code = import_helper.import_module('code') -class TestInteractiveConsole(unittest.TestCase): - - def setUp(self): - self.console = code.InteractiveConsole() - self.mock_sys() +class MockSys: def mock_sys(self): "Mock system environment for InteractiveConsole" @@ -32,6 +28,13 @@ def mock_sys(self): del self.sysmod.ps1 del self.sysmod.ps2 + +class TestInteractiveConsole(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole() + self.mock_sys() + def test_ps1(self): self.infunc.side_effect = EOFError('Finished') self.console.interact() @@ -151,5 +154,21 @@ def test_context_tb(self): self.assertIn(expected, output) +class TestInteractiveConsoleLocalExit(unittest.TestCase, MockSys): + + def setUp(self): + self.console = code.InteractiveConsole(local_exit=True) + self.mock_sys() + + def test_exit(self): + # default exit message + self.infunc.side_effect = ["exit()"] + self.console.interact(banner='') + self.assertEqual(len(self.stderr.method_calls), 2) + err_msg = self.stderr.method_calls[1] + expected = 'now exiting InteractiveConsole...\n' + self.assertEqual(err_msg, ['write', (expected,), {}]) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index c4452e38934cf88..df6e5e4b55f7281 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1283,6 +1283,23 @@ def test_remove_redundant_nop_edge_case(self): def f(): a if (1 if b else c) else d + def test_global_declaration_in_except_used_in_else(self): + # See gh-111123 + code = textwrap.dedent("""\ + def f(): + try: + pass + %s Exception: + global a + else: + print(a) + """) + + g, l = {'a': 5}, {} + for kw in ("except", "except*"): + exec(code % kw, g, l); + + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 5d94ec7cae47069..3dad2567015e249 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -173,6 +173,15 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_trap_no_yield(self): + @contextmanager + def whoo(): + if False: + yield + ctx = whoo() + with self.assertRaises(RuntimeError): + ctx.__enter__() + def test_contextmanager_trap_second_yield(self): @contextmanager def whoo(): @@ -186,6 +195,19 @@ def whoo(): # The "gen" attribute is an implementation detail. self.assertFalse(ctx.gen.gi_suspended) + def test_contextmanager_non_normalised(self): + @contextmanager + def whoo(): + try: + yield + except RuntimeError: + raise SyntaxError + + ctx = whoo() + ctx.__enter__() + with self.assertRaises(SyntaxError): + ctx.__exit__(RuntimeError, None, None) + def test_contextmanager_except(self): state = [] @contextmanager @@ -265,6 +287,25 @@ def test_issue29692(): self.assertEqual(ex.args[0], 'issue29692:Unchained') self.assertIsNone(ex.__cause__) + def test_contextmanager_wrap_runtimeerror(self): + @contextmanager + def woohoo(): + try: + yield + except Exception as exc: + raise RuntimeError(f'caught {exc}') from exc + + with self.assertRaises(RuntimeError): + with woohoo(): + 1 / 0 + + # If the context manager wrapped StopIteration in a RuntimeError, + # we also unwrap it, because we can't tell whether the wrapping was + # done by the generator machinery or by the generator itself. + with self.assertRaises(StopIteration): + with woohoo(): + raise StopIteration + def _create_contextmanager_attribs(self): def attribs(**kw): def decorate(func): @@ -276,6 +317,7 @@ def decorate(func): @attribs(foo='bar') def baz(spam): """Whee!""" + yield return baz def test_contextmanager_attribs(self): @@ -332,8 +374,11 @@ def woohoo(a, *, b): def test_recursive(self): depth = 0 + ncols = 0 @contextmanager def woohoo(): + nonlocal ncols + ncols += 1 nonlocal depth before = depth depth += 1 @@ -347,6 +392,7 @@ def recursive(): recursive() recursive() + self.assertEqual(ncols, 10) self.assertEqual(depth, 0) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 6e12e82a7a00841..6a903ed041ab8d4 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -3212,6 +3212,150 @@ def test_run_doctestsuite_multiple_times(): """ +def test_exception_with_note(note): + """ + >>> test_exception_with_note('Note') + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('Note') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: Text + Note + + >>> test_exception_with_note('''Note + ... multiline + ... example''') + Traceback (most recent call last): + ValueError: Text + Note + multiline + example + + Different note will fail the test: + + >>> def f(x): + ... r''' + ... >>> exc = ValueError('message') + ... >>> exc.add_note('note') + ... >>> raise exc + ... Traceback (most recent call last): + ... ValueError: message + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 5, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + ValueError: message + wrong note + Got: + Traceback (most recent call last): + ... + ValueError: message + note + TestResults(failed=1, attempted=...) + """ + exc = ValueError('Text') + exc.add_note(note) + raise exc + + +def test_exception_with_multiple_notes(): + """ + >>> test_exception_with_multiple_notes() + Traceback (most recent call last): + ... + ValueError: Text + One + Two + """ + exc = ValueError('Text') + exc.add_note('One') + exc.add_note('Two') + raise exc + + +def test_syntax_error_with_note(cls, multiline=False): + """ + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + SyntaxError: error + Note + + >>> test_syntax_error_with_note(SyntaxError) + Traceback (most recent call last): + ... + File "x.py", line 23 + bad syntax + SyntaxError: error + Note + + >>> test_syntax_error_with_note(IndentationError) + Traceback (most recent call last): + ... + IndentationError: error + Note + + >>> test_syntax_error_with_note(TabError, multiline=True) + Traceback (most recent call last): + ... + TabError: error + Note + Line + """ + exc = cls("error", ("x.py", 23, None, "bad syntax")) + exc.add_note('Note\nLine' if multiline else 'Note') + raise exc + + +def test_syntax_error_with_incorrect_expected_note(): + """ + >>> def f(x): + ... r''' + ... >>> exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + ... >>> exc.add_note('note1') + ... >>> exc.add_note('note2') + ... >>> raise exc + ... Traceback (most recent call last): + ... SyntaxError: error + ... wrong note + ... ''' + >>> test = doctest.DocTestFinder().find(f)[0] + >>> doctest.DocTestRunner(verbose=False).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File "...", line 6, in f + Failed example: + raise exc + Expected: + Traceback (most recent call last): + SyntaxError: error + wrong note + Got: + Traceback (most recent call last): + ... + SyntaxError: error + note1 + note2 + TestResults(failed=1, attempted=...) + """ + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(doctest)) tests.addTest(doctest.DocTestSuite()) diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index d2d52ec9a7808fd..e124a7cc7259a29 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -833,7 +833,6 @@ def test_list_all(self): after = set(interpreters.list_all_channels()) self.assertEqual(after, created) - @unittest.expectedFailure # See gh-110318: def test_shareable(self): rch, sch = interpreters.create_channel() @@ -850,6 +849,19 @@ def test_shareable(self): self.assertEqual(rch2, rch) self.assertEqual(sch2, sch) + def test_is_closed(self): + rch, sch = interpreters.create_channel() + rbefore = rch.is_closed + sbefore = sch.is_closed + rch.close() + rafter = rch.is_closed + safter = sch.is_closed + + self.assertFalse(rbefore) + self.assertFalse(sbefore) + self.assertTrue(rafter) + self.assertTrue(safter) + class TestRecvChannelAttrs(TestBase): diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 6ea7e7db2ce1132..4d1766fb16ce9fa 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -7,6 +7,7 @@ import itertools import pprint import random +import re import test.support import test.test_set import types @@ -535,7 +536,10 @@ def test_dataclass_with_repr(self): def test_dataclass_no_repr(self): dc = dataclass3() formatted = pprint.pformat(dc, width=10) - self.assertRegex(formatted, r"") + self.assertRegex( + formatted, + fr"<{re.escape(__name__)}.dataclass3 object at \w+>", + ) def test_recursive_dataclass(self): dc = dataclass4(None) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index e65d9a89ff8a2f5..22b38ac64393832 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -143,18 +143,26 @@ def test_header(self): self.assertTrue(ns.header) def test_randomize(self): - for opt in '-r', '--randomize': + for opt in ('-r', '--randomize'): with self.subTest(opt=opt): ns = self.parse_args([opt]) self.assertTrue(ns.randomize) with os_helper.EnvironmentVarGuard() as env: - env['SOURCE_DATE_EPOCH'] = '1' - + # with SOURCE_DATE_EPOCH + env['SOURCE_DATE_EPOCH'] = '1697839080' ns = self.parse_args(['--randomize']) regrtest = main.Regrtest(ns) self.assertFalse(regrtest.randomize) - self.assertIsNone(regrtest.random_seed) + self.assertIsInstance(regrtest.random_seed, str) + self.assertEqual(regrtest.random_seed, '1697839080') + + # without SOURCE_DATE_EPOCH + del env['SOURCE_DATE_EPOCH'] + ns = self.parse_args(['--randomize']) + regrtest = main.Regrtest(ns) + self.assertTrue(regrtest.randomize) + self.assertIsInstance(regrtest.random_seed, int) def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) @@ -184,34 +192,27 @@ def test_single(self): self.assertTrue(ns.single) self.checkError([opt, '-f', 'foo'], "don't go together") - def test_ignore(self): - for opt in '-i', '--ignore': + def test_match(self): + for opt in '-m', '--match': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.ignore_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', True)]) self.checkError([opt], 'expected one argument') - self.addCleanup(os_helper.unlink, os_helper.TESTFN) - with open(os_helper.TESTFN, "w") as fp: - print('matchfile1', file=fp) - print('matchfile2', file=fp) - - filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--ignorefile', filename]) - self.assertEqual(ns.ignore_tests, - ['matchfile1', 'matchfile2']) - - def test_match(self): - for opt in '-m', '--match': + for opt in '-i', '--ignore': with self.subTest(opt=opt): ns = self.parse_args([opt, 'pattern']) - self.assertEqual(ns.match_tests, ['pattern']) + self.assertEqual(ns.match_tests, [('pattern', False)]) self.checkError([opt], 'expected one argument') - ns = self.parse_args(['-m', 'pattern1', - '-m', 'pattern2']) - self.assertEqual(ns.match_tests, ['pattern1', 'pattern2']) + ns = self.parse_args(['-m', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', True)]) + + ns = self.parse_args(['-m', 'pattern1', '-i', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', True), ('pattern2', False)]) + + ns = self.parse_args(['-i', 'pattern1', '-m', 'pattern2']) + self.assertEqual(ns.match_tests, [('pattern1', False), ('pattern2', True)]) self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, "w") as fp: @@ -219,10 +220,13 @@ def test_match(self): print('matchfile2', file=fp) filename = os.path.abspath(os_helper.TESTFN) - ns = self.parse_args(['-m', 'match', - '--matchfile', filename]) + ns = self.parse_args(['-m', 'match', '--matchfile', filename]) self.assertEqual(ns.match_tests, - ['match', 'matchfile1', 'matchfile2']) + [('match', True), ('matchfile1', True), ('matchfile2', True)]) + + ns = self.parse_args(['-i', 'match', '--ignorefile', filename]) + self.assertEqual(ns.match_tests, + [('match', False), ('matchfile1', False), ('matchfile2', False)]) def test_failfast(self): for opt in '-G', '--failfast': @@ -388,7 +392,13 @@ def check_ci_mode(self, args, use_resources, rerun=True): # Check Regrtest attributes which are more reliable than Namespace # which has an unclear API - regrtest = main.Regrtest(ns) + with os_helper.EnvironmentVarGuard() as env: + # Ignore SOURCE_DATE_EPOCH env var if it's set + if 'SOURCE_DATE_EPOCH' in env: + del env['SOURCE_DATE_EPOCH'] + + regrtest = main.Regrtest(ns) + self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) self.assertTrue(regrtest.randomize) @@ -661,21 +671,26 @@ def list_regex(line_format, tests): state = f'{state} then {new_state}' self.check_line(output, f'Result: {state}', full=True) - def parse_random_seed(self, output): - match = self.regex_search(r'Using random seed ([0-9]+)', output) - randseed = int(match.group(1)) - self.assertTrue(0 <= randseed, randseed) - return randseed + def parse_random_seed(self, output: str) -> str: + match = self.regex_search(r'Using random seed: (.*)', output) + return match.group(1) def run_command(self, args, input=None, exitcode=0, **kw): if not input: input = '' if 'stderr' not in kw: kw['stderr'] = subprocess.STDOUT + + env = kw.pop('env', None) + if env is None: + env = dict(os.environ) + env.pop('SOURCE_DATE_EPOCH', None) + proc = subprocess.run(args, text=True, input=input, stdout=subprocess.PIPE, + env=env, **kw) if proc.returncode != exitcode: msg = ("Command %s failed with exit code %s, but exit code %s expected!\n" @@ -751,7 +766,9 @@ def setUp(self): self.regrtest_args.append('-n') def check_output(self, output): - self.parse_random_seed(output) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) @@ -942,7 +959,7 @@ def test_random(self): test_random = int(match.group(1)) # try to reproduce with the random seed - output = self.run_tests('-r', '--randseed=%s' % randseed, test, + output = self.run_tests('-r', f'--randseed={randseed}', test, exitcode=EXITCODE_NO_TESTS_RAN) randseed2 = self.parse_random_seed(output) self.assertEqual(randseed2, randseed) @@ -953,7 +970,32 @@ def test_random(self): # check that random.seed is used by default output = self.run_tests(test, exitcode=EXITCODE_NO_TESTS_RAN) - self.assertIsInstance(self.parse_random_seed(output), int) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) + + # check SOURCE_DATE_EPOCH (integer) + timestamp = '1697839080' + env = dict(os.environ, SOURCE_DATE_EPOCH=timestamp) + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, timestamp) + self.check_line(output, 'TESTRANDOM: 520') + + # check SOURCE_DATE_EPOCH (string) + env = dict(os.environ, SOURCE_DATE_EPOCH='XYZ') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertEqual(randseed, 'XYZ') + self.check_line(output, 'TESTRANDOM: 22') + + # check SOURCE_DATE_EPOCH (empty string): ignore the env var + env = dict(os.environ, SOURCE_DATE_EPOCH='') + output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN, + env=env) + randseed = self.parse_random_seed(output) + self.assertTrue(randseed.isdigit(), randseed) def test_fromfile(self): # test --fromfile diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index e8ec3b35881fec2..33d0975bda8eaab 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -465,6 +465,44 @@ def test_sitecustomize_executed(self): else: self.fail("sitecustomize not imported automatically") + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 06304dcb4ec7b85..d8ae7b75e181503 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1739,6 +1739,10 @@ def test_buffer_types(self): self.assertEqual(bio.read(), b'bar') bio.write(memoryview(b'baz')) self.assertEqual(bio.read(), b'baz') + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + bio.write(memoryview(noncontig_writable)) def test_error_types(self): bio = ssl.MemoryBIO() diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 6d5353c22764df1..88bc0fd4025a171 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -745,6 +745,7 @@ def test_windows_feature_macros(self): "PyUnicode_AsUCS4Copy", "PyUnicode_AsUTF16String", "PyUnicode_AsUTF32String", + "PyUnicode_AsUTF8", "PyUnicode_AsUTF8AndSize", "PyUnicode_AsUTF8String", "PyUnicode_AsUnicodeEscapeString", diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 97de81677b10bce..41fcc9d5fd3f345 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -557,100 +557,109 @@ def id(self): test_access = Test('test.test_os.FileTests.test_access') test_chdir = Test('test.test_os.Win32ErrorTests.test_chdir') + test_copy = Test('test.test_shutil.TestCopy.test_copy') # Test acceptance - with support.swap_attr(support, '_match_test_func', None): + with support.swap_attr(support, '_test_matchers', ()): # match all support.set_match_tests([]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match all using None - support.set_match_tests(None, None) + support.set_match_tests(None) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the full test identifier - support.set_match_tests([test_access.id()], None) + support.set_match_tests([(test_access.id(), True)]) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # match the module name - support.set_match_tests(['test_os'], None) + support.set_match_tests([('test_os', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) # Test '*' pattern - support.set_match_tests(['test_*'], None) + support.set_match_tests([('test_*', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(['filetests'], None) + support.set_match_tests([('filetests', True)]) self.assertFalse(support.match_test(test_access)) - support.set_match_tests(['FileTests'], None) + support.set_match_tests([('FileTests', True)]) self.assertTrue(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(['*test_os.*.test_*'], None) + support.set_match_tests([('*test_os.*.test_*', True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) # Multiple patterns - support.set_match_tests([test_access.id(), test_chdir.id()], None) + support.set_match_tests([(test_access.id(), True), (test_chdir.id(), True)]) self.assertTrue(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) - support.set_match_tests(['test_access', 'DONTMATCH'], None) + support.set_match_tests([('test_access', True), ('DONTMATCH', True)]) self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # Test rejection - with support.swap_attr(support, '_match_test_func', None): - # match all - support.set_match_tests(ignore_patterns=[]) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - - # match all using None - support.set_match_tests(None, None) - self.assertTrue(support.match_test(test_access)) - self.assertTrue(support.match_test(test_chdir)) - + with support.swap_attr(support, '_test_matchers', ()): # match the full test identifier - support.set_match_tests(None, [test_access.id()]) + support.set_match_tests([(test_access.id(), False)]) self.assertFalse(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) # match the module name - support.set_match_tests(None, ['test_os']) + support.set_match_tests([('test_os', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) # Test '*' pattern - support.set_match_tests(None, ['test_*']) + support.set_match_tests([('test_*', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) # Test case sensitivity - support.set_match_tests(None, ['filetests']) + support.set_match_tests([('filetests', False)]) self.assertTrue(support.match_test(test_access)) - support.set_match_tests(None, ['FileTests']) + support.set_match_tests([('FileTests', False)]) self.assertFalse(support.match_test(test_access)) # Test pattern containing '.' and a '*' metacharacter - support.set_match_tests(None, ['*test_os.*.test_*']) + support.set_match_tests([('*test_os.*.test_*', False)]) self.assertFalse(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) # Multiple patterns - support.set_match_tests(None, [test_access.id(), test_chdir.id()]) + support.set_match_tests([(test_access.id(), False), (test_chdir.id(), False)]) + self.assertFalse(support.match_test(test_access)) + self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) + + support.set_match_tests([('test_access', False), ('DONTMATCH', False)]) self.assertFalse(support.match_test(test_access)) + self.assertTrue(support.match_test(test_chdir)) + + # Test mixed filters + with support.swap_attr(support, '_test_matchers', ()): + support.set_match_tests([('*test_os', False), ('test_access', True)]) + self.assertTrue(support.match_test(test_access)) self.assertFalse(support.match_test(test_chdir)) + self.assertTrue(support.match_test(test_copy)) - support.set_match_tests(None, ['test_access', 'DONTMATCH']) + support.set_match_tests([('*test_os', True), ('test_access', False)]) self.assertFalse(support.match_test(test_access)) self.assertTrue(support.match_test(test_chdir)) + self.assertFalse(support.match_test(test_copy)) @unittest.skipIf(support.is_emscripten, "Unstable in Emscripten") @unittest.skipIf(support.is_wasi, "Unavailable on WASI") diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f5cf9667d564a0d..7ebf9ca1707acd6 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1004,11 +1004,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -2283,8 +2298,14 @@ def test_error_parenthesis(self): def test_error_string_literal(self): - self._check_error("'blech", "unterminated string literal") - self._check_error('"blech', "unterminated string literal") + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) self._check_error("'''blech", "unterminated triple-quoted string literal") self._check_error('"""blech', "unterminated triple-quoted string literal") diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 41b9ebe3374d622..290f4608c5e7397 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1901,19 +1901,9 @@ def test_random_files(self): tempdir = os.path.dirname(__file__) or os.curdir testfiles = glob.glob(os.path.join(glob.escape(tempdir), "test*.py")) - # Tokenize is broken on test_pep3131.py because regular expressions are - # broken on the obscure unicode identifiers in it. *sigh* - # With roundtrip extended to test the 5-tuple mode of untokenize, - # 7 more testfiles fail. Remove them also until the failure is diagnosed. - - testfiles.remove(os.path.join(tempdir, "test_unicode_identifiers.py")) - # TODO: Remove this once we can untokenize PEP 701 syntax testfiles.remove(os.path.join(tempdir, "test_fstring.py")) - for f in ('buffer', 'builtin', 'fileio', 'os', 'platform', 'sys'): - testfiles.remove(os.path.join(tempdir, "test_%s.py") % f) - if not support.is_resource_enabled("cpu"): testfiles = random.sample(testfiles, 10) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 1a1e0a259fd042d..7f60bf4bbc1e756 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2011,13 +2011,13 @@ def test_callable_instance_type_error(self): def f(): pass with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], None]) + isinstance(f, Callable[[], None]) with self.assertRaises(TypeError): - self.assertIsInstance(f, Callable[[], Any]) + isinstance(f, Callable[[], Any]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], None]) + isinstance(None, Callable[[], None]) with self.assertRaises(TypeError): - self.assertNotIsInstance(None, Callable[[], Any]) + isinstance(None, Callable[[], Any]) def test_repr(self): Callable = self.Callable diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index d23eb87696f4061..6af8acc3b0617eb 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -1058,7 +1058,7 @@ def test_assert_called_with_failure_message(self): actual = 'not called.' expected = "mock(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), mock.assert_called_with, 1, '2', 3, bar='foo' @@ -1073,7 +1073,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, '2', 3, bar='foo' @@ -1083,7 +1083,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, bar='foo' @@ -1093,7 +1093,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, 2, 3)" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, 2, 3 @@ -1103,7 +1103,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo()" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth ) @@ -1552,7 +1552,7 @@ def f(x=None): pass '^{}$'.format( re.escape('Calls not found.\n' 'Expected: [call()]\n' - 'Actual: [call(1)]'))) as cm: + ' Actual: [call(1)]'))) as cm: mock.assert_has_calls([call()]) self.assertIsNone(cm.exception.__cause__) @@ -1564,7 +1564,7 @@ def f(x=None): pass 'Error processing expected calls.\n' "Errors: [None, TypeError('too many positional arguments')]\n" "Expected: [call(), call(1, 2)]\n" - 'Actual: [call(1)]'))) as cm: + ' Actual: [call(1)]'))) as cm: mock.assert_has_calls([call(), call(1, 2)]) self.assertIsInstance(cm.exception.__cause__, TypeError) diff --git a/Lib/test/test_zoneinfo/__init__.py b/Lib/test/test_zoneinfo/__init__.py index c3ea567103275de..4b16ecc31156a51 100644 --- a/Lib/test/test_zoneinfo/__init__.py +++ b/Lib/test/test_zoneinfo/__init__.py @@ -1,2 +1,5 @@ -from .test_zoneinfo import * -from .test_zoneinfo_property import * +import os +from test.support import load_package_tests + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/traceback.py b/Lib/traceback.py index 7cc84b9c762aeb0..d3c581f92ba1b87 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -153,14 +153,11 @@ def format_exception_only(exc, /, value=_sentinel): The return value is a list of strings, each ending in a newline. - Normally, the list contains a single string; however, for - SyntaxError exceptions, it contains several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the list. - + The list contains the exception's message, which is + normally a single string; however, for :exc:`SyntaxError` exceptions, it + contains several lines that (when printed) display detailed information + about where the syntax error occurred. Following the message, the list + contains the exception's ``__notes__``. """ if value is _sentinel: value = exc @@ -857,13 +854,13 @@ def format_exception_only(self, *, show_group=False, _depth=0): The return value is a generator of strings, each ending in a newline. - Normally, the generator emits a single string; however, for - SyntaxError exceptions, it emits several lines that (when - printed) display detailed information about where the syntax - error occurred. - - The message indicating which exception occurred is always the last - string in the output. + Generator yields the exception message. + For :exc:`SyntaxError` exceptions, it + also yields (before the exception message) + several lines that (when printed) + display detailed information about where the syntax error occurred. + Following the message, generator also yields + all the exception's ``__notes__``. """ indent = 3 * _depth * ' ' diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index f6c9d6aa6f9a324..2ab6c15e2c079ee 100755 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -161,7 +161,7 @@ def __init__(self, filename=None): label='Help', underline=0) root['menu'] = self.mBar - pane = PanedWindow(orient=HORIZONTAL, sashwidth=5, + pane = PanedWindow(root, orient=HORIZONTAL, sashwidth=5, sashrelief=SOLID, bg='#ddd') pane.add(self.makeTextFrame(pane)) pane.add(self.makeGraphFrame(pane)) diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index e8c8360e0ae13e5..c6b46eea657a21b 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -857,7 +857,7 @@ def _format_mock_call_signature(self, args, kwargs): def _format_mock_failure_message(self, args, kwargs, action='call'): - message = 'expected %s not found.\nExpected: %s\nActual: %s' + message = 'expected %s not found.\nExpected: %s\n Actual: %s' expected_string = self._format_mock_call_signature(args, kwargs) call_args = self.call_args actual_string = self._format_mock_call_signature(*call_args) @@ -960,7 +960,7 @@ def assert_called_with(self, /, *args, **kwargs): if self.call_args is None: expected = self._format_mock_call_signature(args, kwargs) actual = 'not called.' - error_message = ('expected call not found.\nExpected: %s\nActual: %s' + error_message = ('expected call not found.\nExpected: %s\n Actual: %s' % (expected, actual)) raise AssertionError(error_message) @@ -1011,7 +1011,7 @@ def assert_has_calls(self, calls, any_order=False): raise AssertionError( f'{problem}\n' f'Expected: {_CallList(calls)}' - f'{self._calls_repr(prefix="Actual").rstrip(".")}' + f'{self._calls_repr(prefix=" Actual").rstrip(".")}' ) from cause return diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 78b4499cca986ad..32e3a0f55c5d717 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -257,6 +257,8 @@ install_IDLE: rm "$(DESTDIR)$(LIBDEST)/idlelib/config-extensions.def~" ; \ fi touch "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" + chmod -R ugo+rX,go-w "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app" + chmod ugo+x "$(DESTDIR)$(PYTHONAPPSDIR)/IDLE.app/Contents/MacOS/IDLE" $(INSTALLED_PYTHONAPP): install_Python diff --git a/Mac/PythonLauncher/Makefile.in b/Mac/PythonLauncher/Makefile.in index 4c05f26e8358bc3..9decadbdc60f00d 100644 --- a/Mac/PythonLauncher/Makefile.in +++ b/Mac/PythonLauncher/Makefile.in @@ -27,6 +27,8 @@ install: Python\ Launcher.app -test -d "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" && rm -r "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" /bin/cp -r "Python Launcher.app" "$(DESTDIR)$(PYTHONAPPSDIR)" touch "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" + chmod -R ugo+rX,go-w "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app" + chmod ugo+x "$(DESTDIR)$(PYTHONAPPSDIR)/Python Launcher.app/Contents/MacOS/Python Launcher" clean: rm -f *.o "Python Launcher" diff --git a/Misc/HISTORY b/Misc/HISTORY index e66b695f21c9778..b66413277259dc2 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -16169,7 +16169,7 @@ Core and Builtins codec subsystem, for example UTF-8 is turned into utf-8. - Issue #4200: Changed the atexit module to store its state in its - PyModuleDef atexitmodule. This fixes a bug with multiple subinterpeters. + PyModuleDef atexitmodule. This fixes a bug with multiple subinterpreters. - Issue #4237: io.FileIO() was raising invalid warnings caused by insufficient initialization of PyFileIOObject struct members. diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index 3a1694f444616a3..d9cdfbd04c88d46 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -274,7 +274,7 @@ Co-authored-by: Tim Peters Only handle asynchronous exceptions and requests to drop the GIL when returning from a call or on the back edges of loops. Makes sure that -:meth:`__exit__` is always called in with statements, even for interrupts. +:meth:`~object.__exit__` is always called in with statements, even for interrupts. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 7c991e7667b01a3..26c44b6c1af0ed2 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -1720,7 +1720,7 @@ Improve the speed and accuracy of statistics.pvariance(). .. nonce: WI9zQY .. section: Library -Remove :meth:`__getitem__` methods of +Remove :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput`, deprecated since Python 3.9. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 0dc6e945719ec7b..2b9dbd5d63a87e0 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -3592,7 +3592,7 @@ Python 3.5. .. nonce: V8Ou3K .. section: Library -Deprecate :meth:`__getitem__` methods of +Deprecate :meth:`~object.__getitem__` methods of :class:`xml.dom.pulldom.DOMEventStream`, :class:`wsgiref.util.FileWrapper` and :class:`fileinput.FileInput`. diff --git a/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst b/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst index 49b5da32f8f72e2..399054040f2ec75 100644 --- a/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst +++ b/Misc/NEWS.d/next/Build/2023-10-17-01-56-11.gh-issue-85283.V156T2.rst @@ -1,4 +1,5 @@ The ``errno``, ``md5``, ``resource``, ``winsound``, ``_ctypes_test``, -``_scproxy``, ``_stat``, ``_testimportmultiple`` and ``_uuid`` C extensions are -now built with the :ref:`limited C API `. +``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, +``_testimportmultiple`` and ``_uuid`` C extensions are now built with the +:ref:`limited C API `. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst b/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst new file mode 100644 index 000000000000000..446b8b612862f93 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-10-20-15-29-31.gh-issue-111046.2DxQl8.rst @@ -0,0 +1 @@ +For wasi-threads, memory is now exported to fix compatibility issues with some wasm runtimes. diff --git a/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst b/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst new file mode 100644 index 000000000000000..2008dd5438d2b5f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-20-01-42-43.gh-issue-111089.VIrd5q.rst @@ -0,0 +1,2 @@ +The :c:func:`PyUnicode_AsUTF8` function now raises an exception if the +string contains embedded null characters. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst b/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst new file mode 100644 index 000000000000000..fe32e06fe4f0633 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-10-20-18-07-24.gh-issue-111089.RxkyrQ.rst @@ -0,0 +1,2 @@ +Add :c:func:`PyUnicode_AsUTF8` function to the limited C API. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-12-27-02-51-45.gh-issue-100445.C8f6ph.rst b/Misc/NEWS.d/next/Core and Builtins/2022-12-27-02-51-45.gh-issue-100445.C8f6ph.rst new file mode 100644 index 000000000000000..72f38849df9b828 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-12-27-02-51-45.gh-issue-100445.C8f6ph.rst @@ -0,0 +1 @@ +Improve error message for unterminated strings with escapes. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst new file mode 100644 index 000000000000000..f2cebe287db3ee4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-20-23-14-06.gh-issue-111123.jjVc3M.rst @@ -0,0 +1,2 @@ +Fix a bug where a :keyword:`global` declaration in an :keyword:`except` block +is rejected when the global is used in the :keyword:`else` block. diff --git a/Misc/NEWS.d/next/Library/2023-03-22-02-01-30.gh-issue-102895.HiEqaZ.rst b/Misc/NEWS.d/next/Library/2023-03-22-02-01-30.gh-issue-102895.HiEqaZ.rst new file mode 100644 index 000000000000000..20a1a5baccd24b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-22-02-01-30.gh-issue-102895.HiEqaZ.rst @@ -0,0 +1 @@ +Added a parameter ``local_exit`` for :func:`code.interact` to prevent ``exit()`` and ``quit`` from closing ``sys.stdin`` and raise ``SystemExit``. diff --git a/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst b/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst new file mode 100644 index 000000000000000..487bd177d27e31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-19-22-46-34.gh-issue-111092.hgut12.rst @@ -0,0 +1 @@ +Make turtledemo run without default root enabled. diff --git a/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst b/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst new file mode 100644 index 000000000000000..bdec4f4443d80b0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-21-13-57-06.gh-issue-111159.GoHp7s.rst @@ -0,0 +1 @@ +Fix :mod:`doctest` output comparison for exceptions with notes. diff --git a/Misc/NEWS.d/next/Tests/2023-09-15-15-00-14.gh-issue-108747.ql0owS.rst b/Misc/NEWS.d/next/Tests/2023-09-15-15-00-14.gh-issue-108747.ql0owS.rst new file mode 100644 index 000000000000000..ba1fe97871929cc --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-15-15-00-14.gh-issue-108747.ql0owS.rst @@ -0,0 +1,2 @@ +Add unit test for ``usercustomize`` and ``sitecustomize`` hooks from +:class:`site`. diff --git a/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst new file mode 100644 index 000000000000000..7cb79c0cbf29f13 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-16-13-47-24.gh-issue-110918.aFgZK3.rst @@ -0,0 +1,4 @@ +Test case matching patterns specified by options ``--match``, ``--ignore``, +``--matchfile`` and ``--ignorefile`` are now tested in the order of +specification, and the last match determines whether the test case be run or +ignored. diff --git a/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst new file mode 100644 index 000000000000000..45bb0774a9abe3c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-10-21-00-10-36.gh-issue-110932.jktjJU.rst @@ -0,0 +1,2 @@ +Fix regrtest if the ``SOURCE_DATE_EPOCH`` environment variable is defined: +use the variable value as the random seed. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst b/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst new file mode 100644 index 000000000000000..d4c1b56d98ef0ee --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-10-19-21-46-18.gh-issue-110913.CWlPfg.rst @@ -0,0 +1 @@ +WindowsConsoleIO now correctly chunks large buffers without splitting up UTF-8 sequences. diff --git a/Misc/NEWS.d/next/macOS/2023-10-18-01-40-36.gh-issue-111015.NaLI2L.rst b/Misc/NEWS.d/next/macOS/2023-10-18-01-40-36.gh-issue-111015.NaLI2L.rst new file mode 100644 index 000000000000000..4c6eea136554c88 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-10-18-01-40-36.gh-issue-111015.NaLI2L.rst @@ -0,0 +1 @@ +Ensure that IDLE.app and Python Launcher.app are installed with appropriate permissions on macOS builds. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 75c260c8f1b2be8..0601de20fe0f46b 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2478,3 +2478,5 @@ added = '3.13' [function.PySys_AuditTuple] added = '3.13' +[function.PyUnicode_AsUTF8] + added = '3.13' diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index fc08c42bd3574ab..2a0c26a0fec1b66 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1405,7 +1405,7 @@ static PyObject *load_library(PyObject *self, PyObject *args) #ifdef _WIN64 return PyLong_FromVoidPtr(hMod); #else - return Py_BuildValue("i", hMod); + return PyLong_FromLong((int)hMod); #endif } diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d339a8aa798361f..d04d1e973af0303 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -3673,7 +3673,7 @@ _curses_mousemask_impl(PyObject *module, unsigned long newmask) #endif /*[clinic input] -_curses.napms +_curses.napms -> int ms: int Duration in milliseconds. @@ -3682,13 +3682,13 @@ _curses.napms Sleep for specified time. [clinic start generated code]*/ -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms) -/*[clinic end generated code: output=a40a1da2e39ea438 input=20cd3af2b6900f56]*/ +/*[clinic end generated code: output=5f292a6a724491bd input=c6d6e01f2f1df9f7]*/ { PyCursesInitialised; - return Py_BuildValue("i", napms(ms)); + return napms(ms); } diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 684a628806fbb54..9938ed5f8095b57 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -4060,8 +4060,8 @@ static PyObject * timezone_getinitargs(PyDateTime_TimeZone *self, PyObject *Py_UNUSED(ignored)) { if (self->name == NULL) - return Py_BuildValue("(O)", self->offset); - return Py_BuildValue("(OO)", self->offset, self->name); + return PyTuple_Pack(1, self->offset); + return PyTuple_Pack(2, self->offset, self->name); } static PyMethodDef timezone_methods[] = { diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 50b8818aad410b0..6680488b740cfc6 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -134,6 +134,23 @@ char _PyIO_get_console_type(PyObject *path_or_fd) { return m; } +static DWORD +_find_last_utf8_boundary(const char *buf, DWORD len) +{ + /* This function never returns 0, returns the original len instead */ + DWORD count = 1; + if (len == 0 || (buf[len - 1] & 0x80) == 0) { + return len; + } + for (;; count++) { + if (count > 3 || count >= len) { + return len; + } + if ((buf[len - count] & 0xc0) != 0x80) { + return len - count; + } + } +} /*[clinic input] module _io @@ -975,7 +992,7 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, { BOOL res = TRUE; wchar_t *wbuf; - DWORD len, wlen, orig_len, n = 0; + DWORD len, wlen, n = 0; HANDLE handle; if (self->fd == -1) @@ -1007,21 +1024,8 @@ _io__WindowsConsoleIO_write_impl(winconsoleio *self, PyTypeObject *cls, have to reduce and recalculate. */ while (wlen > 32766 / sizeof(wchar_t)) { len /= 2; - orig_len = len; - /* Reduce the length until we hit the final byte of a UTF-8 sequence - * (top bit is unset). Fix for github issue 82052. - */ - while (len > 0 && (((char *)b->buf)[len-1] & 0x80) != 0) - --len; - /* If we hit a length of 0, something has gone wrong. This shouldn't - * be possible, as valid UTF-8 can have at most 3 non-final bytes - * before a final one, and our buffer is way longer than that. - * But to be on the safe side, if we hit this issue we just restore - * the original length and let the console API sort it out. - */ - if (len == 0) { - len = orig_len; - } + /* Fix for github issues gh-110913 and gh-82052. */ + len = _find_last_utf8_boundary(b->buf, len); wlen = MultiByteToWideChar(CP_UTF8, 0, b->buf, len, NULL, 0); } Py_END_ALLOW_THREADS diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index 6b6a9b11c0d990c..1b894ea4c67adc8 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -2,12 +2,6 @@ preserve [clinic start generated code]*/ -#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) -# include "pycore_gc.h" // PyGC_Head -# include "pycore_runtime.h" // _Py_ID() -#endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() - #if defined(HAVE_SHM_OPEN) PyDoc_STRVAR(_posixshmem_shm_open__doc__, @@ -17,69 +11,25 @@ PyDoc_STRVAR(_posixshmem_shm_open__doc__, "Open a shared memory object. Returns a file descriptor (integer)."); #define _POSIXSHMEM_SHM_OPEN_METHODDEF \ - {"shm_open", _PyCFunction_CAST(_posixshmem_shm_open), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_open__doc__}, + {"shm_open", (PyCFunction)(void(*)(void))_posixshmem_shm_open, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_open__doc__}, static int _posixshmem_shm_open_impl(PyObject *module, PyObject *path, int flags, int mode); static PyObject * -_posixshmem_shm_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_open(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 3 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), &_Py_ID(flags), &_Py_ID(mode), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", "flags", "mode", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_open", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[3]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; + static char *_keywords[] = {"path", "flags", "mode", NULL}; PyObject *path; int flags; int mode = 511; int _return_value; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 3, 0, argsbuf); - if (!args) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_open", "argument 'path'", "str", args[0]); - goto exit; - } - path = args[0]; - flags = PyLong_AsInt(args[1]); - if (flags == -1 && PyErr_Occurred()) { - goto exit; - } - if (!noptargs) { - goto skip_optional_pos; - } - mode = PyLong_AsInt(args[2]); - if (mode == -1 && PyErr_Occurred()) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Ui|i:shm_open", _keywords, + &path, &flags, &mode)) goto exit; - } -skip_optional_pos: _return_value = _posixshmem_shm_open_impl(module, path, flags, mode); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; @@ -105,52 +55,21 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ - {"shm_unlink", _PyCFunction_CAST(_posixshmem_shm_unlink), METH_FASTCALL|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, + {"shm_unlink", (PyCFunction)(void(*)(void))_posixshmem_shm_unlink, METH_VARARGS|METH_KEYWORDS, _posixshmem_shm_unlink__doc__}, static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path); static PyObject * -_posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_posixshmem_shm_unlink(PyObject *module, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "shm_unlink", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; + static char *_keywords[] = {"path", NULL}; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "U:shm_unlink", _keywords, + &path)) goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("shm_unlink", "argument 'path'", "str", args[0]); - goto exit; - } - path = args[0]; return_value = _posixshmem_shm_unlink_impl(module, path); exit: @@ -166,4 +85,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=b525631f5915d7f3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=be0661dbed83ea23 input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index f4da9f7ceba6983..c4d1138534d8c58 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,18 +2,17 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ -// clinic/posixshmem.c.h uses internal pycore_modsupport.h API -#ifndef Py_BUILD_CORE_BUILTIN -# define Py_BUILD_CORE_MODULE 1 -#endif +// Need limited C API version 3.13 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +#define Py_LIMITED_API 0x030d0000 #include -// for shm_open() and shm_unlink() +#include // EINTR #ifdef HAVE_SYS_MMAN_H -#include +# include // shm_open(), shm_unlink() #endif + /*[clinic input] module _posixshmem [clinic start generated code]*/ diff --git a/Modules/_opcode.c b/Modules/_opcode.c index dac9c019b249e9a..f0b547795b847a8 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -244,8 +244,7 @@ _opcode_get_nb_ops_impl(PyObject *module) } #define ADD_NB_OP(NUM, STR) \ do { \ - PyObject *pair = Py_BuildValue( \ - "NN", PyUnicode_FromString(#NUM), PyUnicode_FromString(STR)); \ + PyObject *pair = Py_BuildValue("ss", #NUM, STR); \ if (pair == NULL) { \ Py_DECREF(list); \ return NULL; \ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 319ed0c29c7a9b6..ce46c1e69f31310 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -76,15 +76,10 @@ isolation_level_converter(PyObject *str_or_none, const char **result) *result = NULL; } else if (PyUnicode_Check(str_or_none)) { - Py_ssize_t sz; - const char *str = PyUnicode_AsUTF8AndSize(str_or_none, &sz); + const char *str = PyUnicode_AsUTF8(str_or_none); if (str == NULL) { return 0; } - if (strlen(str) != (size_t)sz) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - return 0; - } const char *level = get_isolation_level(str); if (level == NULL) { diff --git a/Modules/_testcapi/getargs.c b/Modules/_testcapi/getargs.c index 86d8f5984dd3e5e..a06cdf08da1385c 100644 --- a/Modules/_testcapi/getargs.c +++ b/Modules/_testcapi/getargs.c @@ -359,68 +359,83 @@ getargs_K(PyObject *self, PyObject *args) static PyObject * test_k_code(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *tuple, *num; - unsigned long value; - - tuple = PyTuple_New(1); + PyObject *tuple = PyTuple_New(1); if (tuple == NULL) { return NULL; } /* a number larger than ULONG_MAX even on 64-bit platforms */ - num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); + PyObject *num = PyLong_FromString("FFFFFFFFFFFFFFFFFFFFFFFF", NULL, 16); if (num == NULL) { - return NULL; + goto error; } - value = PyLong_AsUnsignedLongMask(num); - if (value != ULONG_MAX) { + unsigned long value = PyLong_AsUnsignedLongMask(num); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + Py_DECREF(num); + goto error; + } + else if (value != ULONG_MAX) { + Py_DECREF(num); PyErr_SetString(PyExc_AssertionError, "test_k_code: " "PyLong_AsUnsignedLongMask() returned wrong value for long 0xFFF...FFF"); - return NULL; + goto error; } PyTuple_SET_ITEM(tuple, 0, num); value = 0; if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; + goto error; } if (value != ULONG_MAX) { PyErr_SetString(PyExc_AssertionError, "test_k_code: k code returned wrong value for long 0xFFF...FFF"); - return NULL; + goto error; } - Py_DECREF(num); + Py_DECREF(tuple); // also clears `num` + tuple = PyTuple_New(1); + if (tuple == NULL) { + return NULL; + } num = PyLong_FromString("-FFFFFFFF000000000000000042", NULL, 16); if (num == NULL) { - return NULL; + goto error; } value = PyLong_AsUnsignedLongMask(num); - if (value != (unsigned long)-0x42) { + if (value == (unsigned long)-1 && PyErr_Occurred()) { + Py_DECREF(num); + goto error; + } + else if (value != (unsigned long)-0x42) { + Py_DECREF(num); PyErr_SetString(PyExc_AssertionError, "test_k_code: " "PyLong_AsUnsignedLongMask() returned wrong value for long -0xFFF..000042"); - return NULL; + goto error; } PyTuple_SET_ITEM(tuple, 0, num); value = 0; if (!PyArg_ParseTuple(tuple, "k:test_k_code", &value)) { - return NULL; + goto error; } if (value != (unsigned long)-0x42) { PyErr_SetString(PyExc_AssertionError, "test_k_code: k code returned wrong value for long -0xFFF..000042"); - return NULL; + goto error; } Py_DECREF(tuple); Py_RETURN_NONE; + +error: + Py_DECREF(tuple); + return NULL; } static PyObject * @@ -710,51 +725,56 @@ getargs_et_hash(PyObject *self, PyObject *args) static PyObject * test_L_code(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *tuple, *num; - long long value; - - tuple = PyTuple_New(1); + PyObject *tuple = PyTuple_New(1); if (tuple == NULL) { return NULL; } - num = PyLong_FromLong(42); + PyObject *num = PyLong_FromLong(42); if (num == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(tuple, 0, num); - value = -1; + long long value = -1; if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; + goto error; } if (value != 42) { PyErr_SetString(PyExc_AssertionError, "test_L_code: L code returned wrong value for long 42"); - return NULL; + goto error; } - Py_DECREF(num); + Py_DECREF(tuple); // also clears `num` + tuple = PyTuple_New(1); + if (tuple == NULL) { + return NULL; + } num = PyLong_FromLong(42); if (num == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(tuple, 0, num); value = -1; if (!PyArg_ParseTuple(tuple, "L:test_L_code", &value)) { - return NULL; + goto error; } if (value != 42) { PyErr_SetString(PyExc_AssertionError, "test_L_code: L code returned wrong value for int 42"); - return NULL; + goto error; } Py_DECREF(tuple); Py_RETURN_NONE; + +error: + Py_DECREF(tuple); + return NULL; } /* Test the s and z codes for PyArg_ParseTuple. @@ -771,7 +791,7 @@ test_s_code(PyObject *self, PyObject *Py_UNUSED(ignored)) PyObject *obj = PyUnicode_Decode("t\xeate", strlen("t\xeate"), "latin-1", NULL); if (obj == NULL) { - return NULL; + goto error; } PyTuple_SET_ITEM(tuple, 0, obj); @@ -781,15 +801,19 @@ test_s_code(PyObject *self, PyObject *Py_UNUSED(ignored)) */ char *value; if (!PyArg_ParseTuple(tuple, "s:test_s_code1", &value)) { - return NULL; + goto error; } if (!PyArg_ParseTuple(tuple, "z:test_s_code2", &value)) { - return NULL; + goto error; } Py_DECREF(tuple); Py_RETURN_NONE; + +error: + Py_DECREF(tuple); + return NULL; } static PyObject * diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index d52d88a65d86fcc..a10183dddeca98c 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -634,7 +634,7 @@ unicode_asutf8andsize(PyObject *self, PyObject *args) NULLABLE(unicode); s = PyUnicode_AsUTF8AndSize(unicode, &size); if (s == NULL) { - assert(size == UNINITIALIZED_SIZE); + assert(size == -1); return NULL; } diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 33b49dacaa946e3..82a0c827deeddf9 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -75,10 +75,18 @@ test_lock_two_threads(PyObject *self, PyObject *obj) assert(test_data.m.v == 1); PyThread_start_new_thread(lock_thread, &test_data); - while (!_Py_atomic_load_int(&test_data.started)) { - pysleep(10); - } - pysleep(10); // allow some time for the other thread to try to lock + + // wait up to two seconds for the lock_thread to attempt to lock "m" + int iters = 0; + uint8_t v; + do { + pysleep(10); // allow some time for the other thread to try to lock + v = _Py_atomic_load_uint8_relaxed(&test_data.m.v); + assert(v == 1 || v == 3); + iters++; + } while (v != 3 && iters < 200); + + // both the "locked" and the "has parked" bits should be set assert(test_data.m.v == 3); PyMutex_Unlock(&test_data.m); diff --git a/Modules/_winapi.c b/Modules/_winapi.c index eec33499b983fe7..8c48b6f3ec6ef67 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -212,7 +212,7 @@ class DWORD_return_converter(CReturnConverter): self.declare(data) self.err_occurred_if("_return_value == PY_DWORD_MAX", data) data.return_conversion.append( - 'return_value = Py_BuildValue("k", _return_value);\n') + 'return_value = PyLong_FromUnsignedLong(_return_value);\n') class LPVOID_return_converter(CReturnConverter): type = 'LPVOID' @@ -223,7 +223,7 @@ class LPVOID_return_converter(CReturnConverter): data.return_conversion.append( 'return_value = HANDLE_TO_PYNUM(_return_value);\n') [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=011ee0c3a2244bfe]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ef52a757a1830d92]*/ #include "clinic/_winapi.c.h" diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index b618592bf002798..11fe8cd01fc47b4 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -223,8 +223,8 @@ static PyTypeObject * add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, struct xid_class_registry *classes) { - PyTypeObject *cls = (PyTypeObject *)PyType_FromMetaclass( - NULL, mod, spec, NULL); + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, spec, NULL); if (cls == NULL) { return NULL; } @@ -402,6 +402,7 @@ typedef struct { PyTypeObject *recv_channel_type; /* heap types */ + PyTypeObject *ChannelInfoType; PyTypeObject *ChannelIDType; PyTypeObject *XIBufferViewType; @@ -440,7 +441,12 @@ _get_current_module_state(void) static int traverse_module_state(module_state *state, visitproc visit, void *arg) { + /* external types */ + Py_VISIT(state->send_channel_type); + Py_VISIT(state->recv_channel_type); + /* heap types */ + Py_VISIT(state->ChannelInfoType); Py_VISIT(state->ChannelIDType); Py_VISIT(state->XIBufferViewType); @@ -457,10 +463,12 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) static int clear_module_state(module_state *state) { + /* external types */ Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); /* heap types */ + Py_CLEAR(state->ChannelInfoType); if (state->ChannelIDType != NULL) { (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); } @@ -2088,6 +2096,236 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, } +/* channel info */ + +struct channel_info { + struct { + // 1: closed; -1: closing + int closed; + struct { + Py_ssize_t nsend_only; // not released + Py_ssize_t nsend_only_released; + Py_ssize_t nrecv_only; // not released + Py_ssize_t nrecv_only_released; + Py_ssize_t nboth; // not released + Py_ssize_t nboth_released; + Py_ssize_t nboth_send_released; + Py_ssize_t nboth_recv_released; + } all; + struct { + // 1: associated; -1: released + int send; + int recv; + } cur; + } status; + Py_ssize_t count; +}; + +static int +_channel_get_info(_channels *channels, int64_t cid, struct channel_info *info) +{ + int err = 0; + *info = (struct channel_info){0}; + + // Get the current interpreter. + PyInterpreterState *interp = _get_current_interp(); + if (interp == NULL) { + return -1; + } + Py_ssize_t interpid = PyInterpreterState_GetID(interp); + + // Hold the global lock until we're done. + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + // Find the channel. + _channelref *ref = _channelref_find(channels->head, cid, NULL); + if (ref == NULL) { + err = ERR_CHANNEL_NOT_FOUND; + goto finally; + } + _channel_state *chan = ref->chan; + + // Check if open. + if (chan == NULL) { + info->status.closed = 1; + goto finally; + } + if (!chan->open) { + assert(chan->queue->count == 0); + info->status.closed = 1; + goto finally; + } + if (chan->closing != NULL) { + assert(chan->queue->count > 0); + info->status.closed = -1; + } + else { + info->status.closed = 0; + } + + // Get the number of queued objects. + info->count = chan->queue->count; + + // Get the ends statuses. + assert(info->status.cur.send == 0); + assert(info->status.cur.recv == 0); + _channelend *send = chan->ends->send; + while (send != NULL) { + if (send->interpid == interpid) { + info->status.cur.send = send->open ? 1 : -1; + } + + if (send->open) { + info->status.all.nsend_only += 1; + } + else { + info->status.all.nsend_only_released += 1; + } + send = send->next; + } + _channelend *recv = chan->ends->recv; + while (recv != NULL) { + if (recv->interpid == interpid) { + info->status.cur.recv = recv->open ? 1 : -1; + } + + // XXX This is O(n*n). Why do we have 2 linked lists? + _channelend *send = chan->ends->send; + while (send != NULL) { + if (send->interpid == recv->interpid) { + break; + } + send = send->next; + } + if (send == NULL) { + if (recv->open) { + info->status.all.nrecv_only += 1; + } + else { + info->status.all.nrecv_only_released += 1; + } + } + else { + if (recv->open) { + if (send->open) { + info->status.all.nboth += 1; + info->status.all.nsend_only -= 1; + } + else { + info->status.all.nboth_recv_released += 1; + info->status.all.nsend_only_released -= 1; + } + } + else { + if (send->open) { + info->status.all.nboth_send_released += 1; + info->status.all.nsend_only -= 1; + } + else { + info->status.all.nboth_released += 1; + info->status.all.nsend_only_released -= 1; + } + } + } + recv = recv->next; + } + +finally: + PyThread_release_lock(channels->mutex); + return err; +} + +PyDoc_STRVAR(channel_info_doc, +"ChannelInfo\n\ +\n\ +A named tuple of a channel's state."); + +static PyStructSequence_Field channel_info_fields[] = { + {"open", "both ends are open"}, + {"closing", "send is closed, recv is non-empty"}, + {"closed", "both ends are closed"}, + {"count", "queued objects"}, + + {"num_interp_send", "interpreters bound to the send end"}, + {"num_interp_send_released", + "interpreters bound to the send end and released"}, + + {"num_interp_recv", "interpreters bound to the send end"}, + {"num_interp_recv_released", + "interpreters bound to the send end and released"}, + + {"num_interp_both", "interpreters bound to both ends"}, + {"num_interp_both_released", + "interpreters bound to both ends and released_from_both"}, + {"num_interp_both_send_released", + "interpreters bound to both ends and released_from_the send end"}, + {"num_interp_both_recv_released", + "interpreters bound to both ends and released_from_the recv end"}, + + {"send_associated", "current interpreter is bound to the send end"}, + {"send_released", "current interpreter *was* bound to the send end"}, + {"recv_associated", "current interpreter is bound to the recv end"}, + {"recv_released", "current interpreter *was* bound to the recv end"}, + {0} +}; + +static PyStructSequence_Desc channel_info_desc = { + .name = MODULE_NAME ".ChannelInfo", + .doc = channel_info_doc, + .fields = channel_info_fields, + .n_in_sequence = 8, +}; + +static PyObject * +new_channel_info(PyObject *mod, struct channel_info *info) +{ + module_state *state = get_module_state(mod); + if (state == NULL) { + return NULL; + } + + assert(state->ChannelInfoType != NULL); + PyObject *self = PyStructSequence_New(state->ChannelInfoType); + if (self == NULL) { + return NULL; + } + + int pos = 0; +#define SET_BOOL(val) \ + PyStructSequence_SET_ITEM(self, pos++, \ + Py_NewRef(val ? Py_True : Py_False)) +#define SET_COUNT(val) \ + do { \ + PyObject *obj = PyLong_FromLongLong(val); \ + if (obj == NULL) { \ + Py_CLEAR(info); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(self, pos++, obj); \ + } while(0) + SET_BOOL(info->status.closed == 0); + SET_BOOL(info->status.closed == -1); + SET_BOOL(info->status.closed == 1); + SET_COUNT(info->count); + SET_COUNT(info->status.all.nsend_only); + SET_COUNT(info->status.all.nsend_only_released); + SET_COUNT(info->status.all.nrecv_only); + SET_COUNT(info->status.all.nrecv_only_released); + SET_COUNT(info->status.all.nboth); + SET_COUNT(info->status.all.nboth_released); + SET_COUNT(info->status.all.nboth_send_released); + SET_COUNT(info->status.all.nboth_recv_released); + SET_BOOL(info->status.cur.send == 1); + SET_BOOL(info->status.cur.send == -1); + SET_BOOL(info->status.cur.recv == 1); + SET_BOOL(info->status.cur.recv == -1); +#undef SET_COUNT +#undef SET_BOOL + assert(!PyErr_Occurred()); + return self; +} + + /* ChannelID class */ typedef struct channelid { @@ -3079,6 +3317,33 @@ Close the channel for the current interpreter. 'send' and 'recv'\n\ (bool) may be used to indicate the ends to close. By default both\n\ ends are closed. Closing an already closed end is a noop."); +static PyObject * +channelsmod_get_info(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"cid", NULL}; + struct channel_id_converter_data cid_data = { + .module = self, + }; + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&:_get_info", kwlist, + channel_id_converter, &cid_data)) { + return NULL; + } + int64_t cid = cid_data.cid; + + struct channel_info info; + int err = _channel_get_info(&_globals.channels, cid, &info); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + return new_channel_info(self, &info); +} + +PyDoc_STRVAR(channelsmod_get_info_doc, +"get_info(cid)\n\ +\n\ +Return details about the channel."); + static PyObject * channelsmod__channel_id(PyObject *self, PyObject *args, PyObject *kwds) { @@ -3143,6 +3408,8 @@ static PyMethodDef module_functions[] = { METH_VARARGS | METH_KEYWORDS, channelsmod_close_doc}, {"release", _PyCFunction_CAST(channelsmod_release), METH_VARARGS | METH_KEYWORDS, channelsmod_release_doc}, + {"get_info", _PyCFunction_CAST(channelsmod_get_info), + METH_VARARGS | METH_KEYWORDS, channelsmod_get_info_doc}, {"_channel_id", _PyCFunction_CAST(channelsmod__channel_id), METH_VARARGS | METH_KEYWORDS, NULL}, {"_register_end_types", _PyCFunction_CAST(channelsmod__register_end_types), @@ -3179,6 +3446,15 @@ module_exec(PyObject *mod) /* Add other types */ + // ChannelInfo + state->ChannelInfoType = PyStructSequence_NewType(&channel_info_desc); + if (state->ChannelInfoType == NULL) { + goto error; + } + if (PyModule_AddType(mod, state->ChannelInfoType) < 0) { + goto error; + } + // ChannelID state->ChannelIDType = add_new_type( mod, &channelid_typespec, _channelid_shared, xid_classes); @@ -3186,12 +3462,14 @@ module_exec(PyObject *mod) goto error; } + // XIBufferView state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL, xid_classes); if (state->XIBufferViewType == NULL) { goto error; } + // Register external types. if (register_builtin_xid_types(xid_classes) < 0) { goto error; } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 3f14dd13988da9a..97e550197a2a8d4 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -816,7 +816,7 @@ zoneinfo_ZoneInfo__unpickle_impl(PyTypeObject *type, PyTypeObject *cls, /*[clinic end generated code: output=556712fc709deecb input=6ac8c73eed3de316]*/ { if (from_cache) { - PyObject *val_args = Py_BuildValue("(O)", key); + PyObject *val_args = PyTuple_Pack(1, key); if (val_args == NULL) { return NULL; } diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index dd9f1ab6841d014..f7e0aaf7b23649f 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -3163,7 +3163,7 @@ PyDoc_STRVAR(_curses_napms__doc__, #define _CURSES_NAPMS_METHODDEF \ {"napms", (PyCFunction)_curses_napms, METH_O, _curses_napms__doc__}, -static PyObject * +static int _curses_napms_impl(PyObject *module, int ms); static PyObject * @@ -3171,12 +3171,17 @@ _curses_napms(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; int ms; + int _return_value; ms = PyLong_AsInt(arg); if (ms == -1 && PyErr_Occurred()) { goto exit; } - return_value = _curses_napms_impl(module, ms); + _return_value = _curses_napms_impl(module, ms); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromLong((long)_return_value); exit: return return_value; @@ -4313,4 +4318,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_USE_DEFAULT_COLORS_METHODDEF #define _CURSES_USE_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_USE_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=24ad16254d1eef9c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=96887782374f070a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index ae8cc1d6283217a..3a3231c051ef713 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -568,7 +568,7 @@ _winapi_GetExitCodeProcess(PyObject *module, PyObject *arg) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -595,7 +595,7 @@ _winapi_GetLastError(PyObject *module, PyObject *Py_UNUSED(ignored)) if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -1305,7 +1305,7 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P if ((_return_value == PY_DWORD_MAX) && PyErr_Occurred()) { goto exit; } - return_value = Py_BuildValue("k", _return_value); + return_value = PyLong_FromUnsignedLong(_return_value); exit: return return_value; @@ -1479,4 +1479,4 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=aaf29735c47f55fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e1a9908bb82a6379 input=a9049054013a1b77]*/ diff --git a/Modules/overlapped.c b/Modules/overlapped.c index e23db22dadb18bb..fd40e91d0f50c44 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -599,8 +599,7 @@ _overlapped_FormatMessage_impl(PyObject *module, DWORD code) if (n) { while (iswspace(lpMsgBuf[n-1])) --n; - lpMsgBuf[n] = L'\0'; - res = Py_BuildValue("u", lpMsgBuf); + res = PyUnicode_FromWideChar(lpMsgBuf, n); } else { res = PyUnicode_FromFormat("unknown error code %u", code); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 650ae4bbd68656f..f9797f6d7081709 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11276,9 +11276,9 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, done: #if !defined(HAVE_LARGEFILE_SUPPORT) - return Py_BuildValue("l", sbytes); + return PyLong_FromLong(sbytes); #else - return Py_BuildValue("L", sbytes); + return PyLong_FromLongLong(sbytes); #endif #else @@ -11291,7 +11291,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); } #endif off_t offset; @@ -11312,7 +11312,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, return (!async_err) ? posix_error() : NULL; if (offset >= st.st_size) { - return Py_BuildValue("i", 0); + return PyLong_FromLong(0); } // On illumos specifically sendfile() may perform a partial write but @@ -11338,7 +11338,7 @@ os_sendfile_impl(PyObject *module, int out_fd, int in_fd, PyObject *offobj, } while (ret < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); if (ret < 0) return (!async_err) ? posix_error() : NULL; - return Py_BuildValue("n", ret); + return PyLong_FromSsize_t(ret); #endif } #endif /* HAVE_SENDFILE */ diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index bd24523eac830b3..21579a80dd7f703 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -895,7 +895,7 @@ static PyObject * pyexpat_xmlparser_GetBase_impl(xmlparseobject *self) /*[clinic end generated code: output=2886cb21f9a8739a input=918d71c38009620e]*/ { - return Py_BuildValue("z", XML_GetBase(self->itself)); + return conv_string_to_unicode(XML_GetBase(self->itself)); } /*[clinic input] @@ -1585,7 +1585,7 @@ static PyObject * pyexpat_ErrorString_impl(PyObject *module, long code) /*[clinic end generated code: output=2feae50d166f2174 input=cc67de010d9e62b3]*/ { - return Py_BuildValue("z", XML_ErrorString((int)code)); + return conv_string_to_unicode(XML_ErrorString((int)code)); } /* List of methods defined in the module */ diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index bc5cdf0ab6b5f2d..2932d94858afdec 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -657,7 +657,7 @@ signal_strsignal_impl(PyObject *module, int signalnum) Py_RETURN_NONE; #endif - return Py_BuildValue("s", res); + return PyUnicode_FromString(res); } #ifdef HAVE_SIGINTERRUPT diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3261a14a053dc89..250856930702210 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3499,13 +3499,14 @@ type_new_set_doc(PyTypeObject *type) return 0; } - const char *doc_str = PyUnicode_AsUTF8(doc); + Py_ssize_t doc_size; + const char *doc_str = PyUnicode_AsUTF8AndSize(doc, &doc_size); if (doc_str == NULL) { return -1; } // Silently truncate the docstring if it contains a null byte - Py_ssize_t size = strlen(doc_str) + 1; + Py_ssize_t size = doc_size + 1; char *tp_doc = (char *)PyObject_Malloc(size); if (tp_doc == NULL) { PyErr_NoMemory(); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 33cbc987d432825..80b19567c63d208 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3820,24 +3820,37 @@ PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *psize) { if (!PyUnicode_Check(unicode)) { PyErr_BadArgument(); + if (psize) { + *psize = -1; + } return NULL; } if (PyUnicode_UTF8(unicode) == NULL) { if (unicode_fill_utf8(unicode) == -1) { + if (psize) { + *psize = -1; + } return NULL; } } - if (psize) + if (psize) { *psize = PyUnicode_UTF8_LENGTH(unicode); + } return PyUnicode_UTF8(unicode); } const char * PyUnicode_AsUTF8(PyObject *unicode) { - return PyUnicode_AsUTF8AndSize(unicode, NULL); + Py_ssize_t size; + const char *utf8 = PyUnicode_AsUTF8AndSize(unicode, &size); + if (utf8 != NULL && strlen(utf8) != (size_t)size) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + return NULL; + } + return utf8; } /* diff --git a/PC/python3dll.c b/PC/python3dll.c index d12889f44d65b60..7f5d97ae4dc83f9 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -661,6 +661,7 @@ EXPORT_FUNC(PyUnicode_AsUCS4Copy) EXPORT_FUNC(PyUnicode_AsUnicodeEscapeString) EXPORT_FUNC(PyUnicode_AsUTF16String) EXPORT_FUNC(PyUnicode_AsUTF32String) +EXPORT_FUNC(PyUnicode_AsUTF8) EXPORT_FUNC(PyUnicode_AsUTF8AndSize) EXPORT_FUNC(PyUnicode_AsUTF8String) EXPORT_FUNC(PyUnicode_AsWideChar) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 1a01bb0352a7b11..2ba24a2c2405f22 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -972,6 +972,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t int quote = c; int quote_size = 1; /* 1 or 3 */ int end_quote_size = 0; + int has_escaped_quote = 0; /* Nodes of type STRING, especially multi line strings must be handled differently in order to get both @@ -1037,8 +1038,18 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } else { - _PyTokenizer_syntaxerror(tok, "unterminated string literal (detected at" - " line %d)", start); + if (has_escaped_quote) { + _PyTokenizer_syntaxerror( + tok, + "unterminated string literal (detected at line %d); " + "perhaps you escaped the end quote?", + start + ); + } else { + _PyTokenizer_syntaxerror( + tok, "unterminated string literal (detected at line %d)", start + ); + } if (c != '\n') { tok->done = E_EOLS; } @@ -1052,6 +1063,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t end_quote_size = 0; if (c == '\\') { c = tok_nextc(tok); /* skip escaped char */ + if (c == quote) { /* but record whether the escaped char was a quote */ + has_escaped_quote = 1; + } if (c == '\r') { c = tok_nextc(tok); } diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 41f48eba08afc41..04d7ae6eaafbc0e 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -142,15 +142,6 @@ check_complexity(PyObject *obj, Py_ssize_t limit) } return limit; } - else if (PyFrozenSet_Check(obj)) { - Py_ssize_t i = 0; - PyObject *item; - Py_hash_t hash; - limit -= PySet_GET_SIZE(obj); - while (limit >= 0 && _PySet_NextEntry(obj, &i, &item, &hash)) { - limit = check_complexity(item, limit); - } - } return limit; } @@ -174,9 +165,8 @@ safe_multiply(PyObject *v, PyObject *w) return NULL; } } - else if (PyLong_Check(v) && (PyTuple_Check(w) || PyFrozenSet_Check(w))) { - Py_ssize_t size = PyTuple_Check(w) ? PyTuple_GET_SIZE(w) : - PySet_GET_SIZE(w); + else if (PyLong_Check(v) && PyTuple_Check(w)) { + Py_ssize_t size = PyTuple_GET_SIZE(w); if (size) { long n = PyLong_AsLong(v); if (n < 0 || n > MAX_COLLECTION_SIZE / size) { @@ -198,8 +188,7 @@ safe_multiply(PyObject *v, PyObject *w) } } else if (PyLong_Check(w) && - (PyTuple_Check(v) || PyFrozenSet_Check(v) || - PyUnicode_Check(v) || PyBytes_Check(v))) + (PyTuple_Check(v) || PyUnicode_Check(v) || PyBytes_Check(v))) { return safe_multiply(w, v); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b915c063d0b456e..db4991662b8bb1c 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1277,7 +1277,9 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCodeObject *co = _PyAST_Compile(mod, interactive_filename, flags, -1, arena); if (co == NULL) { - Py_DECREF(interactive_filename); + if (interactive_src) { + Py_DECREF(interactive_filename); + } return NULL; } diff --git a/Python/symtable.c b/Python/symtable.c index 75ea9e902f4381a..da7fec0ee7cf0cf 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -1813,14 +1813,14 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) break; case Try_kind: VISIT_SEQ(st, stmt, s->v.Try.body); - VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, excepthandler, s->v.Try.handlers); + VISIT_SEQ(st, stmt, s->v.Try.orelse); VISIT_SEQ(st, stmt, s->v.Try.finalbody); break; case TryStar_kind: VISIT_SEQ(st, stmt, s->v.TryStar.body); - VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers); + VISIT_SEQ(st, stmt, s->v.TryStar.orelse); VISIT_SEQ(st, stmt, s->v.TryStar.finalbody); break; case Assert_kind: diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 5dd7900851550a0..8a4bbcbe7f64b12 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1432,67 +1432,64 @@ def parser_body( deprecated_keywords[i] = p has_optional_kw = (max(pos_only, min_pos) + min_kw_only < len(converters) - int(vararg != NO_VARARG)) - if vararg == NO_VARARG: - # FIXME: refactor the code to not declare args_declaration - # if limited_capi is true - if not limited_capi: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywords()') - args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only - ) - nargs = "nargs" - else: - if not limited_capi: - clinic.add_include('pycore_modsupport.h', - '_PyArg_UnpackKeywordsWithVararg()') - args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( - min_pos, - max_pos, - min_kw_only, - vararg - ) - nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" if limited_capi: parser_code = None fastcall = False - - elif fastcall: - flags = "METH_FASTCALL|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS - argname_fmt = 'args[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); - if (!args) {{ - goto exit; - }} - """ % args_declaration, indent=4)] else: - # positional-or-keyword arguments - flags = "METH_VARARGS|METH_KEYWORDS" - parser_prototype = self.PARSER_PROTOTYPE_KEYWORD - argname_fmt = 'fastargs[%d]' - declarations = declare_parser(f, clinic=clinic, - limited_capi=clinic.limited_capi) - declarations += "\nPyObject *argsbuf[%s];" % len(converters) - declarations += "\nPyObject * const *fastargs;" - declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" - if has_optional_kw: - declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) - parser_code = [normalize_snippet(""" - fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); - if (!fastargs) {{ - goto exit; - }} - """ % args_declaration, indent=4)] + if vararg == NO_VARARG: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywords()') + args_declaration = "_PyArg_UnpackKeywords", "%s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only + ) + nargs = "nargs" + else: + clinic.add_include('pycore_modsupport.h', + '_PyArg_UnpackKeywordsWithVararg()') + args_declaration = "_PyArg_UnpackKeywordsWithVararg", "%s, %s, %s, %s" % ( + min_pos, + max_pos, + min_kw_only, + vararg + ) + nargs = f"Py_MIN(nargs, {max_pos})" if max_pos else "0" + + if fastcall: + flags = "METH_FASTCALL|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_FASTCALL_KEYWORDS + argname_fmt = 'args[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [normalize_snippet(""" + args = %s(args, nargs, NULL, kwnames, &_parser, %s, argsbuf); + if (!args) {{ + goto exit; + }} + """ % args_declaration, indent=4)] + else: + # positional-or-keyword arguments + flags = "METH_VARARGS|METH_KEYWORDS" + parser_prototype = self.PARSER_PROTOTYPE_KEYWORD + argname_fmt = 'fastargs[%d]' + declarations = declare_parser(f, clinic=clinic, + limited_capi=clinic.limited_capi) + declarations += "\nPyObject *argsbuf[%s];" % len(converters) + declarations += "\nPyObject * const *fastargs;" + declarations += "\nPy_ssize_t nargs = PyTuple_GET_SIZE(args);" + if has_optional_kw: + declarations += "\nPy_ssize_t noptargs = %s + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - %d;" % (nargs, min_pos + min_kw_only) + parser_code = [normalize_snippet(""" + fastargs = %s(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, %s, argsbuf); + if (!fastargs) {{ + goto exit; + }} + """ % args_declaration, indent=4)] if requires_defining_class: flags = 'METH_METHOD|' + flags diff --git a/configure b/configure index c87f518382f2ece..3bf92ddb8fcb40c 100755 --- a/configure +++ b/configure @@ -9268,10 +9268,15 @@ then : # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. as_fn_append CFLAGS " -target wasm32-wasi-threads -pthread" as_fn_append CFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -target wasm32-wasi-threads -pthread" as_fn_append LDFLAGS_NODIST " -Wl,--import-memory" + as_fn_append LDFLAGS_NODIST " -Wl,--export-memory" as_fn_append LDFLAGS_NODIST " -Wl,--max-memory=10485760" fi diff --git a/configure.ac b/configure.ac index cd69f0ede544963..daa59b2737bae99 100644 --- a/configure.ac +++ b/configure.ac @@ -2162,10 +2162,15 @@ AS_CASE([$ac_sys_system], # without this, configure fails to find pthread_create, sem_init, # etc because they are only available in the sysroot for # wasm32-wasi-threads. + # Note: wasi-threads requires --import-memory. + # Note: wasi requires --export-memory. + # Note: --export-memory is implicit unless --import-memory is given + # Note: this requires LLVM >= 16. AS_VAR_APPEND([CFLAGS], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([CFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -target wasm32-wasi-threads -pthread"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--import-memory"]) + AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--export-memory"]) AS_VAR_APPEND([LDFLAGS_NODIST], [" -Wl,--max-memory=10485760"]) ])